lee/renderdoc
1
0
Fork 0
mirror of https://github.com/baldurk/renderdoc.git synced 2026-05-05 09:30:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add easy support for self-hosted captures on windows only

Browse Source 
* By renaming the renderdoc and renderdoccmd projects to something else
  (say 'selfhost' and 'selfhostcmd') then they can be used to capture
  renderdocui and the replaying that's happening.
* Only supported on development builds and might break down, but it's
  handy to have as an easy to enable option.
* There's also a couple of handy python functions exposed -
  renderdoc.StaticExports.StartSelfHostCapture(string dllname)
  renderdoc.StaticExports.EndSelfHostCapture(string dllname)
  which can be used to start and stop the capture around e.g. a shader
  debug operation or a pixel history operation or something similar.
This commit is contained in:
baldurk
2016-10-25 19:37:37 +02:00
parent aa5c55e4e4
commit f368a65843
8 changed files with 93 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
renderdoc/os/win32/comexport.def
-1
View File
@@ -1,4 +1,3 @@
LIBRARY RENDERDOC
EXPORTS
DllGetClassObject PRIVATE
DllCanUnloadNow PRIVATE
renderdoc/os/win32/win32_callstack.cpp
+1 -1
View File
@@ -148,7 +148,7 @@ static bool InitDbgHelp()
else
{
wchar_t path[MAX_PATH] = {0};
GetModuleFileNameW(GetModuleHandleA("renderdoc.dll"), path, MAX_PATH - 1);
GetModuleFileNameW(GetModuleHandleA(STRINGIZE(RDOC_DLL_FILE) ".dll"), path, MAX_PATH - 1);
wchar_t *slash = wcsrchr(path, '\\');
renderdoc/os/win32/win32_hook.cpp
+1 -1
View File
@@ -175,7 +175,7 @@ struct CachedHookData
// Also we exclude ourselves here - just in case the application has already loaded
// renderdoc.dll, or tries to load it.
if(strstr(lowername, "fraps") || strstr(lowername, "gameoverlayrenderer") ||
strstr(lowername, "renderdoc.dll") == lowername)
strstr(lowername, STRINGIZE(RDOC_DLL_FILE) ".dll") == lowername)
return;
// set module pointer if we are hooking exports from this module
renderdoc/os/win32/win32_libentry.cpp
+5 -4
View File
@@ -43,14 +43,15 @@ static BOOL add_hooks()
if(f.find(L"dllhost.exe") != wstring::npos || f.find(L"explorer.exe") != wstring::npos)
{
#ifndef _RELEASE
OutputDebugStringA("Hosting renderdoc.dll in shell process\n");
OutputDebugStringA("Hosting " STRINGIZE(RDOC_DLL_FILE) ".dll in shell process\n");
#endif
return TRUE;
}
if(f.find(L"renderdoccmd.exe") != wstring::npos ||
f.find(L"renderdocui.vshost.exe") != wstring::npos ||
f.find(L"qrenderdoc.exe") != wstring::npos || f.find(L"renderdocui.exe") != wstring::npos)
if(f.find(CONCAT(L, STRINGIZE(RDOC_DLL_FILE)) L"cmd.exe") != wstring::npos ||
f.find(CONCAT(L, STRINGIZE(RDOC_DLL_FILE)) L"ui.vshost.exe") != wstring::npos ||
f.find(L"q" CONCAT(L, STRINGIZE(RDOC_DLL_FILE)) L".exe") != wstring::npos ||
f.find(CONCAT(L, STRINGIZE(RDOC_DLL_FILE)) L"ui.exe") != wstring::npos)
{
RDCDEBUG("Not creating hooks - in replay app");
renderdoc/os/win32/win32_process.cpp
+11 -7
View File
@@ -348,7 +348,7 @@ void InjectFunctionCall(HANDLE hProcess, uintptr_t renderdoc_remote, const char
RDCDEBUG("Injecting call to %s", funcName);
HMODULE renderdoc_local = GetModuleHandleA("renderdoc.dll");
HMODULE renderdoc_local = GetModuleHandleA(STRINGIZE(RDOC_DLL_FILE) ".dll");
uintptr_t func_local = (uintptr_t)GetProcAddress(renderdoc_local, funcName);
@@ -480,7 +480,8 @@ uint32_t Process::InjectIntoProcess(uint32_t pid, EnvironmentModification *env,
RDCLOG("Injecting renderdoc into process %lu", pid);
wchar_t renderdocPath[MAX_PATH] = {0};
GetModuleFileNameW(GetModuleHandleA("renderdoc.dll"), &renderdocPath[0], MAX_PATH - 1);
GetModuleFileNameW(GetModuleHandleA(STRINGIZE(RDOC_DLL_FILE) ".dll"), &renderdocPath[0],
MAX_PATH - 1);
BOOL isWow64 = FALSE;
BOOL success = IsWow64Process(hProcess, &isWow64);
@@ -652,13 +653,13 @@ uint32_t Process::InjectIntoProcess(uint32_t pid, EnvironmentModification *env,
InjectDLL(hProcess, renderdocPath);
uintptr_t loc = FindRemoteDLL(pid, L"renderdoc.dll");
uintptr_t loc = FindRemoteDLL(pid, CONCAT(L, STRINGIZE(RDOC_DLL_FILE)) L".dll");
uint32_t controlident = 0;
if(loc == 0)
{
RDCERR("Can't locate renderdoc.dll in remote PID %d", pid);
RDCERR("Can't locate " STRINGIZE(RDOC_DLL_FILE) ".dll in remote PID %d", pid);
}
else
{
@@ -732,11 +733,13 @@ uint32_t Process::LaunchAndInjectIntoProcess(const char *app, const char *workin
const char *logfile, const CaptureOptions *opts,
bool waitForExit)
{
void *func = GetProcAddress(GetModuleHandleA("renderdoc.dll"), "RENDERDOC_SetLogFile");
void *func =
GetProcAddress(GetModuleHandleA(STRINGIZE(RDOC_DLL_FILE) ".dll"), "RENDERDOC_SetLogFile");
if(func == NULL)
{
RDCERR("Can't find required export function in renderdoc.dll - corrupted/missing file?");
RDCERR("Can't find required export function in " STRINGIZE(
RDOC_DLL_FILE) ".dll - corrupted/missing file?");
return 0;
}
@@ -771,7 +774,8 @@ void Process::StartGlobalHook(const char *pathmatch, const char *logfile, const
return;
wchar_t renderdocPath[MAX_PATH] = {0};
GetModuleFileNameW(GetModuleHandleA("renderdoc.dll"), &renderdocPath[0], MAX_PATH - 1);
GetModuleFileNameW(GetModuleHandleA(STRINGIZE(RDOC_DLL_FILE) ".dll"), &renderdocPath[0],
MAX_PATH - 1);
wchar_t *slash = wcsrchr(renderdocPath, L'\\');
renderdoc/renderdoc.vcxproj
+4 -4
View File
@@ -106,7 +106,7 @@
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>RENDERDOC_EXPORTS;RENDERDOC_PLATFORM_WIN32;USE_BREAKPAD;WIN32;RELEASE;NDEBUG;_WINDOWS;_USRDLL;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>RENDERDOC_EXPORTS;RENDERDOC_PLATFORM_WIN32;USE_BREAKPAD;WIN32;RELEASE;NDEBUG;_WINDOWS;_USRDLL;RDOC_DLL_FILE=$(ProjectName);%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PrecompiledHeaderFile>
</PrecompiledHeaderFile>
<AdditionalOptions>/wd4100 /wd4512</AdditionalOptions>
@@ -135,7 +135,7 @@
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>RENDERDOC_EXPORTS;RENDERDOC_PLATFORM_WIN32;USE_BREAKPAD;WIN64;WIN32;RELEASE;NDEBUG;_WINDOWS;_USRDLL;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>RENDERDOC_EXPORTS;RENDERDOC_PLATFORM_WIN32;USE_BREAKPAD;WIN64;WIN32;RELEASE;NDEBUG;_WINDOWS;_USRDLL;RDOC_DLL_FILE=$(ProjectName);%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PrecompiledHeaderFile>
</PrecompiledHeaderFile>
<AdditionalOptions>/wd4100 /wd4512</AdditionalOptions>
@@ -163,7 +163,7 @@
<Optimization>Disabled</Optimization>
<FunctionLevelLinking>false</FunctionLevelLinking>
<IntrinsicFunctions>false</IntrinsicFunctions>
<PreprocessorDefinitions>RENDERDOC_EXPORTS;RENDERDOC_PLATFORM_WIN32;WIN32;NDEBUG;_WINDOWS;_USRDLL;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>RENDERDOC_EXPORTS;RENDERDOC_PLATFORM_WIN32;WIN32;NDEBUG;_WINDOWS;_USRDLL;RDOC_DLL_FILE=$(ProjectName);%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PrecompiledHeaderFile>
</PrecompiledHeaderFile>
<AdditionalOptions>/wd4100 /wd4512</AdditionalOptions>
@@ -195,7 +195,7 @@
<Optimization>Disabled</Optimization>
<FunctionLevelLinking>false</FunctionLevelLinking>
<IntrinsicFunctions>false</IntrinsicFunctions>
<PreprocessorDefinitions>RENDERDOC_EXPORTS;RENDERDOC_PLATFORM_WIN32;WIN64;WIN32;NDEBUG;_WINDOWS;_USRDLL;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>RENDERDOC_EXPORTS;RENDERDOC_PLATFORM_WIN32;WIN64;WIN32;NDEBUG;_WINDOWS;_USRDLL;RDOC_DLL_FILE=$(ProjectName);%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PrecompiledHeaderFile>
</PrecompiledHeaderFile>
<AdditionalOptions>/wd4100 /wd4512</AdditionalOptions>
renderdoc/replay/entry_points.cpp
+46
View File
@@ -545,3 +545,49 @@ extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_BecomeRemoteServer(const ch
RenderDoc::Inst().BecomeRemoteServer(listenhost, (uint16_t)port, *killReplay);
}
extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_StartSelfHostCapture(const char *dllname)
{
void *module = Process::LoadModule(dllname);
if(module == NULL)
return;
pRENDERDOC_GetAPI get =
(pRENDERDOC_GetAPI)Process::GetFunctionAddress(module, "RENDERDOC_GetAPI");
if(get == NULL)
return;
RENDERDOC_API_1_0_0 *rdoc = NULL;
get(eRENDERDOC_API_Version_1_0_0, (void **)&rdoc);
if(rdoc == NULL)
return;
rdoc->StartFrameCapture(NULL, NULL);
}
extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_EndSelfHostCapture(const char *dllname)
{
void *module = Process::LoadModule(dllname);
if(module == NULL)
return;
pRENDERDOC_GetAPI get =
(pRENDERDOC_GetAPI)Process::GetFunctionAddress(module, "RENDERDOC_GetAPI");
if(get == NULL)
return;
RENDERDOC_API_1_0_0 *rdoc = NULL;
get(eRENDERDOC_API_Version_1_0_0, (void **)&rdoc);
if(rdoc == NULL)
return;
rdoc->EndFrameCapture(NULL, NULL);
}
renderdocui/Interop/StaticExports.cs
+25 -1
View File
@@ -46,7 +46,7 @@ namespace renderdoc
public ReplayCreateStatus Status;
}
class StaticExports
public class StaticExports
{
[DllImport("renderdoc.dll", CharSet = CharSet.Unicode, CallingConvention = CallingConvention.Cdecl)]
private static extern ReplaySupport RENDERDOC_SupportLocalReplay(IntPtr logfile, IntPtr outdriver, IntPtr outident);
@@ -110,6 +110,30 @@ namespace renderdoc
[DllImport("renderdoc.dll", CharSet = CharSet.Unicode, CallingConvention = CallingConvention.Cdecl)]
private static extern void RENDERDOC_FreeEnvironmentModificationList(IntPtr mem);
[DllImport("renderdoc.dll", CharSet = CharSet.Unicode, CallingConvention = CallingConvention.Cdecl)]
private static extern void RENDERDOC_StartSelfHostCapture(IntPtr dllname);
[DllImport("renderdoc.dll", CharSet = CharSet.Unicode, CallingConvention = CallingConvention.Cdecl)]
private static extern void RENDERDOC_EndSelfHostCapture(IntPtr dllname);
public static void StartSelfHostCapture(string dllname)
{
IntPtr mem = CustomMarshal.MakeUTF8String(dllname);
RENDERDOC_StartSelfHostCapture(mem);
CustomMarshal.Free(mem);
}
public static void EndSelfHostCapture(string dllname)
{
IntPtr mem = CustomMarshal.MakeUTF8String(dllname);
RENDERDOC_EndSelfHostCapture(mem);
CustomMarshal.Free(mem);
}
public static ReplaySupport SupportLocalReplay(string logfile, out string driverName, out string recordMachineIdent)
{
IntPtr name_mem = CustomMarshal.Alloc(typeof(templated_array));