lee/renderdoc
1
0
Fork 0
mirror of https://github.com/baldurk/renderdoc.git synced 2026-05-13 13:30:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fail with error msg if RenderDoc layer is explicitly loaded in instance

Browse Source 
* This is not supported and users who are confused about how RenderDoc works or
  how layers function may try to enable it and get into problems with
  themselves.
This commit is contained in:
baldurk
2023-12-05 11:21:17 +00:00
parent 8cd1bf9eb1
commit b71084d946
1 changed files with 48 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
renderdoc/driver/vulkan/wrappers/vk_device_funcs.cpp
+48
View File
@@ -590,6 +590,54 @@ VkResult WrappedVulkan::vkCreateInstance(const VkInstanceCreateInfo *pCreateInfo
VkInstanceCreateInfo modifiedCreateInfo;
modifiedCreateInfo = *pCreateInfo;
for(uint32_t i = 0; i < modifiedCreateInfo.enabledLayerCount; i++)
{
if(rdcstr(modifiedCreateInfo.ppEnabledLayerNames[i]) == RENDERDOC_VULKAN_LAYER_NAME)
{
// see if any debug report callbacks were passed in the pNext chain
VkDebugReportCallbackCreateInfoEXT *report =
(VkDebugReportCallbackCreateInfoEXT *)pCreateInfo->pNext;
rdcstr msg =
"RenderDoc's layer should NEVER be activated manually. Do not include it in "
"vkCreateInstance's instance layers.";
RDCERR("%s", msg.c_str());
while(report)
{
if(report->sType == VK_STRUCTURE_TYPE_DEBUG_REPORT_CALLBACK_CREATE_INFO_EXT)
report->pfnCallback(VK_DEBUG_REPORT_ERROR_BIT_EXT, VK_DEBUG_REPORT_OBJECT_TYPE_INSTANCE_EXT,
0, 1, 1, "RDOC", msg.c_str(), report->pUserData);
report = (VkDebugReportCallbackCreateInfoEXT *)report->pNext;
}
// or debug utils callbacks
VkDebugUtilsMessengerCreateInfoEXT *messenger =
(VkDebugUtilsMessengerCreateInfoEXT *)pCreateInfo->pNext;
VkDebugUtilsMessengerCallbackDataEXT messengerData = {};
messengerData.messageIdNumber = 1;
messengerData.pMessageIdName = NULL;
messengerData.pMessage = msg.c_str();
messengerData.sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_MESSENGER_CALLBACK_DATA_EXT;
while(messenger)
{
if(messenger->sType == VK_STRUCTURE_TYPE_DEBUG_UTILS_MESSENGER_CREATE_INFO_EXT)
messenger->pfnUserCallback(VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT,
VK_DEBUG_UTILS_MESSAGE_TYPE_VALIDATION_BIT_EXT, &messengerData,
messenger->pUserData);
messenger = (VkDebugUtilsMessengerCreateInfoEXT *)messenger->pNext;
}
return VK_ERROR_INITIALIZATION_FAILED;
}
}
for(uint32_t i = 0; i < modifiedCreateInfo.enabledExtensionCount; i++)
{
if(!IsSupportedExtension(modifiedCreateInfo.ppEnabledExtensionNames[i]))