From 7db90232f9973bb390ba82d86dae5c6bfceace3c Mon Sep 17 00:00:00 2001 From: baldurk Date: Mon, 6 May 2019 15:48:50 +0100 Subject: [PATCH] Allow creating completely headless outputs and reading back their output * This is useful when writing automated tests that want to test the output of rendering which only happens to outputs, such as mesh rendering, or could potentially be bypassed with direct readback like GetTextureData vs rendering a texture. --- renderdoc/api/replay/renderdoc_replay.h | 38 +- renderdoc/core/image_viewer.cpp | 8 + renderdoc/core/replay_proxy.h | 10 + renderdoc/driver/d3d11/d3d11_outputwindow.cpp | 216 +++++-- renderdoc/driver/d3d11/d3d11_replay.h | 2 + renderdoc/driver/d3d12/d3d12_outputwindow.cpp | 261 +++++++-- renderdoc/driver/d3d12/d3d12_replay.cpp | 3 +- renderdoc/driver/d3d12/d3d12_replay.h | 3 + renderdoc/driver/gl/cgl_platform.cpp | 2 +- renderdoc/driver/gl/egl_platform.cpp | 3 +- renderdoc/driver/gl/gl_outputwindow.cpp | 110 +++- renderdoc/driver/gl/gl_replay.h | 3 + renderdoc/driver/gl/glx_platform.cpp | 2 +- renderdoc/driver/gl/wgl_platform.cpp | 9 +- renderdoc/driver/vulkan/vk_android.cpp | 7 + renderdoc/driver/vulkan/vk_apple.cpp | 7 + renderdoc/driver/vulkan/vk_ggp.cpp | 7 + renderdoc/driver/vulkan/vk_linux.cpp | 7 + renderdoc/driver/vulkan/vk_outputwindow.cpp | 554 +++++++++++------- renderdoc/driver/vulkan/vk_posix.cpp | 3 + renderdoc/driver/vulkan/vk_rendermesh.cpp | 2 +- renderdoc/driver/vulkan/vk_rendertexture.cpp | 2 +- renderdoc/driver/vulkan/vk_replay.cpp | 4 +- renderdoc/driver/vulkan/vk_replay.h | 4 +- renderdoc/driver/vulkan/vk_win32.cpp | 10 + renderdoc/replay/replay_controller.h | 2 + renderdoc/replay/replay_driver.h | 2 + renderdoc/replay/replay_output.cpp | 38 +- util/test/tests/D3D11/D3D11_Overlay_Test.py | 4 +- util/test/tests/D3D12/D3D12_Overlay_Test.py | 4 +- util/test/tests/GL/GL_Overlay_Test.py | 4 +- util/test/tests/Vulkan/VK_Overlay_Test.py | 4 +- 32 files changed, 1009 insertions(+), 326 deletions(-) diff --git a/renderdoc/api/replay/renderdoc_replay.h b/renderdoc/api/replay/renderdoc_replay.h index 3b85edce8..4b8ef3526 100644 --- a/renderdoc/api/replay/renderdoc_replay.h +++ b/renderdoc/api/replay/renderdoc_replay.h @@ -310,7 +310,12 @@ DOCUMENT(R"(Specifies a windowing system to use for creating an output window. .. data:: Unknown - No windowing data is passed and no native window is described. + Unknown window type, no windowing data is passed and no native window is described. + +.. data:: Headless + + The windowing data doesn't describe a real window but a virtual area, allowing all normal output + rendering to happen off-screen. See :func:`CreateHeadlessWindowingData`. .. data:: Win32 @@ -337,6 +342,7 @@ DOCUMENT(R"(Specifies a windowing system to use for creating an output window. enum class WindowingSystem : uint32_t { Unknown, + Headless, Win32, Xlib, XCB, @@ -381,6 +387,11 @@ struct WindowingData union { + struct + { + int32_t width, height; + } headless; + struct { HWND window; @@ -417,14 +428,20 @@ DECLARE_REFLECTION_ENUM(WindowingData); DOCUMENT(R"(Create a :class:`WindowingData` for no backing window, it will be headless. +:param int width: The initial width for this virtual window. +:param int height: The initial height for this virtual window. + :return: A :class:`WindowingData` corresponding to an 'empty' backing window. :rtype: WindowingData )"); -inline const WindowingData CreateHeadlessWindowingData() +inline const WindowingData CreateHeadlessWindowingData(int32_t width, int32_t height) { WindowingData ret = {}; - ret.system = WindowingSystem::Unknown; + ret.system = WindowingSystem::Headless; + + ret.headless.width = width > 0 ? width : 1; + ret.headless.height = height > 0 ? height : 1; return ret; } @@ -649,6 +666,21 @@ which is useful for operations like picking vertices that depends on the output )"); virtual void SetDimensions(int32_t width, int32_t height) = 0; + DOCUMENT(R"(Read the output texture back as byte data. Primarily useful for headless outputs where +the output data is not displayed anywhere natively. + +:return: The output texture data as tightly packed RGB 3-byte data. +:rtype: ``bytes`` +)"); + virtual bytebuf ReadbackOutputTexture() = 0; + + DOCUMENT(R"(Retrieve the current dimensions of the output. + +:return: The current width and height of the output. +:rtype: ``pair`` of two ``int`` +)"); + virtual rdcpair GetDimensions() = 0; + DOCUMENT( "Clear and release all thumbnails associated with this output. See :meth:`AddThumbnail`."); virtual void ClearThumbnails() = 0; diff --git a/renderdoc/core/image_viewer.cpp b/renderdoc/core/image_viewer.cpp index 0f07f6d0e..eb44bd59d 100644 --- a/renderdoc/core/image_viewer.cpp +++ b/renderdoc/core/image_viewer.cpp @@ -81,10 +81,18 @@ public: } void DestroyOutputWindow(uint64_t id) { m_Proxy->DestroyOutputWindow(id); } bool CheckResizeOutputWindow(uint64_t id) { return m_Proxy->CheckResizeOutputWindow(id); } + void SetOutputWindowDimensions(uint64_t id, int32_t w, int32_t h) + { + m_Proxy->SetOutputWindowDimensions(id, w, h); + } void GetOutputWindowDimensions(uint64_t id, int32_t &w, int32_t &h) { m_Proxy->GetOutputWindowDimensions(id, w, h); } + void GetOutputWindowData(uint64_t id, bytebuf &retData) + { + m_Proxy->GetOutputWindowData(id, retData); + } void ClearOutputWindowColor(uint64_t id, FloatVector col) { m_Proxy->ClearOutputWindowColor(id, col); diff --git a/renderdoc/core/replay_proxy.h b/renderdoc/core/replay_proxy.h index ec828e3bf..487f45088 100644 --- a/renderdoc/core/replay_proxy.h +++ b/renderdoc/core/replay_proxy.h @@ -195,6 +195,16 @@ public: if(m_Proxy) return m_Proxy->GetOutputWindowDimensions(id, w, h); } + void SetOutputWindowDimensions(uint64_t id, int32_t w, int32_t h) + { + if(m_Proxy) + m_Proxy->SetOutputWindowDimensions(id, w, h); + } + void GetOutputWindowData(uint64_t id, bytebuf &retData) + { + if(m_Proxy) + m_Proxy->GetOutputWindowData(id, retData); + } void ClearOutputWindowColor(uint64_t id, FloatVector col) { if(m_Proxy) diff --git a/renderdoc/driver/d3d11/d3d11_outputwindow.cpp b/renderdoc/driver/d3d11/d3d11_outputwindow.cpp index adfddcd80..f04638dd0 100644 --- a/renderdoc/driver/d3d11/d3d11_outputwindow.cpp +++ b/renderdoc/driver/d3d11/d3d11_outputwindow.cpp @@ -29,13 +29,37 @@ void D3D11Replay::OutputWindow::MakeRTV() { ID3D11Texture2D *texture = NULL; - HRESULT hr = swap->GetBuffer(0, __uuidof(ID3D11Texture2D), (void **)&texture); - if(FAILED(hr)) + HRESULT hr = S_OK; + + if(swap) { - RDCERR("Failed to get swap chain buffer, HRESULT: %s", ToStr(hr).c_str()); - SAFE_RELEASE(texture); - return; + hr = swap->GetBuffer(0, __uuidof(ID3D11Texture2D), (void **)&texture); + + if(FAILED(hr)) + { + RDCERR("Failed to get swap chain buffer, HRESULT: %s", ToStr(hr).c_str()); + SAFE_RELEASE(texture); + return; + } + } + else + { + D3D11_TEXTURE2D_DESC texDesc; + + texDesc.ArraySize = 1; + texDesc.BindFlags = D3D11_BIND_SHADER_RESOURCE | D3D11_BIND_RENDER_TARGET; + texDesc.CPUAccessFlags = 0; + texDesc.MipLevels = 1; + texDesc.MiscFlags = 0; + texDesc.SampleDesc.Count = 1; + texDesc.SampleDesc.Quality = 0; + texDesc.Usage = D3D11_USAGE_DEFAULT; + texDesc.Width = width; + texDesc.Height = height; + texDesc.Format = DXGI_FORMAT_R8G8B8A8_UNORM_SRGB; + + hr = dev->CreateTexture2D(&texDesc, NULL, &texture); } hr = dev->CreateRenderTargetView(texture, NULL, &rtv); @@ -52,13 +76,19 @@ void D3D11Replay::OutputWindow::MakeRTV() void D3D11Replay::OutputWindow::MakeDSV() { - ID3D11Texture2D *texture = NULL; - HRESULT hr = swap->GetBuffer(0, __uuidof(ID3D11Texture2D), (void **)&texture); + if(!rtv) + return; - if(FAILED(hr)) + ID3D11Texture2D *texture = NULL; { - RDCERR("Failed to get swap chain buffer, HRESULT: %s", ToStr(hr).c_str()); - SAFE_RELEASE(texture); + ID3D11Resource *res = NULL; + rtv->GetResource(&res); + texture = (ID3D11Texture2D *)res; + } + + if(!texture) + { + RDCERR("Failed to get swap chain buffer from RTV"); return; } @@ -70,7 +100,7 @@ void D3D11Replay::OutputWindow::MakeDSV() texDesc.Format = DXGI_FORMAT_D24_UNORM_S8_UINT; texDesc.BindFlags = D3D11_BIND_DEPTH_STENCIL; - hr = dev->CreateTexture2D(&texDesc, NULL, &texture); + HRESULT hr = dev->CreateTexture2D(&texDesc, NULL, &texture); if(FAILED(hr)) { @@ -95,38 +125,49 @@ void D3D11Replay::OutputWindow::MakeDSV() uint64_t D3D11Replay::MakeOutputWindow(WindowingData window, bool depth) { - RDCASSERT(window.system == WindowingSystem::Win32, window.system); + RDCASSERT(window.system == WindowingSystem::Win32 || window.system == WindowingSystem::Headless, + window.system); - OutputWindow outw; - outw.wnd = window.win32.window; + DXGI_SWAP_CHAIN_DESC swapDesc = {}; + OutputWindow outw = {}; outw.dev = m_pDevice; - DXGI_SWAP_CHAIN_DESC swapDesc; - RDCEraseEl(swapDesc); - - RECT rect; - GetClientRect(outw.wnd, &rect); - - swapDesc.BufferCount = 2; - swapDesc.BufferDesc.Format = DXGI_FORMAT_R8G8B8A8_UNORM_SRGB; - outw.width = swapDesc.BufferDesc.Width = rect.right - rect.left; - outw.height = swapDesc.BufferDesc.Height = rect.bottom - rect.top; - swapDesc.BufferUsage = DXGI_USAGE_RENDER_TARGET_OUTPUT; - swapDesc.SampleDesc.Count = depth ? 4 : 1; - swapDesc.SampleDesc.Quality = 0; - swapDesc.OutputWindow = outw.wnd; - swapDesc.Windowed = TRUE; - swapDesc.SwapEffect = DXGI_SWAP_EFFECT_DISCARD; - swapDesc.Flags = 0; - - HRESULT hr = S_OK; - - hr = m_pFactory->CreateSwapChain(m_pDevice, &swapDesc, &outw.swap); - - if(FAILED(hr)) + if(window.system == WindowingSystem::Win32) { - RDCERR("Failed to create swap chain for HWND, HRESULT: %s", ToStr(hr).c_str()); - return 0; + outw.wnd = window.win32.window; + + RECT rect = {}; + GetClientRect(outw.wnd, &rect); + + swapDesc.BufferCount = 2; + swapDesc.BufferDesc.Format = DXGI_FORMAT_R8G8B8A8_UNORM_SRGB; + outw.width = swapDesc.BufferDesc.Width = rect.right - rect.left; + outw.height = swapDesc.BufferDesc.Height = rect.bottom - rect.top; + swapDesc.BufferUsage = DXGI_USAGE_RENDER_TARGET_OUTPUT; + swapDesc.SampleDesc.Count = depth ? 4 : 1; + swapDesc.SampleDesc.Quality = 0; + swapDesc.OutputWindow = outw.wnd; + swapDesc.Windowed = TRUE; + swapDesc.SwapEffect = DXGI_SWAP_EFFECT_DISCARD; + swapDesc.Flags = 0; + + HRESULT hr = S_OK; + + hr = m_pFactory->CreateSwapChain(m_pDevice, &swapDesc, &outw.swap); + + if(FAILED(hr)) + { + RDCERR("Failed to create swap chain for HWND, HRESULT: %s", ToStr(hr).c_str()); + return 0; + } + } + else + { + outw.width = window.headless.width; + outw.height = window.headless.height; + + outw.wnd = NULL; + outw.swap = NULL; } outw.MakeRTV(); @@ -215,6 +256,100 @@ void D3D11Replay::GetOutputWindowDimensions(uint64_t id, int32_t &w, int32_t &h) h = m_OutputWindows[id].height; } +void D3D11Replay::SetOutputWindowDimensions(uint64_t id, int32_t w, int32_t h) +{ + if(id == 0 || m_OutputWindows.find(id) == m_OutputWindows.end()) + return; + + OutputWindow &outw = m_OutputWindows[id]; + + // can't resize an output with an actual window backing + if(outw.wnd) + return; + + SAFE_RELEASE(outw.rtv); + SAFE_RELEASE(outw.dsv); + + outw.width = w; + outw.height = h; + + outw.MakeRTV(); + outw.MakeDSV(); +} + +void D3D11Replay::GetOutputWindowData(uint64_t id, bytebuf &retData) +{ + if(id == 0 || m_OutputWindows.find(id) == m_OutputWindows.end()) + return; + + OutputWindow &outw = m_OutputWindows[id]; + + if(!outw.rtv) + return; + + ID3D11Texture2D *texture = NULL; + { + ID3D11Resource *res = NULL; + outw.rtv->GetResource(&res); + texture = (ID3D11Texture2D *)res; + } + + if(!texture) + { + RDCERR("Couldn't get backbuffer texture"); + return; + } + + ID3D11Texture2D *readback = NULL; + + D3D11_TEXTURE2D_DESC texDesc; + texture->GetDesc(&texDesc); + + texDesc.CPUAccessFlags = D3D11_CPU_ACCESS_READ; + texDesc.BindFlags = 0; + texDesc.Usage = D3D11_USAGE_STAGING; + + HRESULT hr = m_pDevice->CreateTexture2D(&texDesc, NULL, &readback); + + if(FAILED(hr)) + { + RDCERR("Couldn't create staging texture for readback, HRESULT: %s", ToStr(hr).c_str()); + SAFE_RELEASE(texture); + return; + } + + ID3D11DeviceContext *ctx = m_pDevice->GetImmediateContext(); + + ctx->CopyResource(readback, texture); + + SAFE_RELEASE(texture); + + D3D11_MAPPED_SUBRESOURCE mapped = {}; + ctx->Map(readback, 0, D3D11_MAP_READ, 0, &mapped); + + retData.resize(outw.width * outw.height * 3); + + byte *src = (byte *)mapped.pData; + byte *dst = retData.data(); + + for(int32_t row = 0; row < outw.height; row++) + { + for(int32_t x = 0; x < outw.width; x++) + { + dst[x * 3 + 0] = src[x * 4 + 0]; + dst[x * 3 + 1] = src[x * 4 + 1]; + dst[x * 3 + 2] = src[x * 4 + 2]; + } + + src += mapped.RowPitch; + dst += outw.width * 3; + } + + ctx->Unmap(readback, 0); + + SAFE_RELEASE(readback); +} + void D3D11Replay::ClearOutputWindowColor(uint64_t id, FloatVector col) { if(id == 0 || m_OutputWindows.find(id) == m_OutputWindows.end()) @@ -259,6 +394,9 @@ bool D3D11Replay::IsOutputWindowVisible(uint64_t id) if(id == 0 || m_OutputWindows.find(id) == m_OutputWindows.end()) return false; + if(!m_OutputWindows[id].wnd) + return true; + return (IsWindowVisible(m_OutputWindows[id].wnd) == TRUE); } diff --git a/renderdoc/driver/d3d11/d3d11_replay.h b/renderdoc/driver/d3d11/d3d11_replay.h index 2d3afc18e..05a08fe26 100644 --- a/renderdoc/driver/d3d11/d3d11_replay.h +++ b/renderdoc/driver/d3d11/d3d11_replay.h @@ -154,6 +154,8 @@ public: void DestroyOutputWindow(uint64_t id); bool CheckResizeOutputWindow(uint64_t id); void GetOutputWindowDimensions(uint64_t id, int32_t &w, int32_t &h); + void SetOutputWindowDimensions(uint64_t id, int32_t w, int32_t h); + void GetOutputWindowData(uint64_t id, bytebuf &retData); void ClearOutputWindowColor(uint64_t id, FloatVector col); void ClearOutputWindowDepth(uint64_t id, float depth, uint8_t stencil); void BindOutputWindow(uint64_t id, bool depth); diff --git a/renderdoc/driver/d3d12/d3d12_outputwindow.cpp b/renderdoc/driver/d3d12/d3d12_outputwindow.cpp index 16187b2d5..3e927c0f9 100644 --- a/renderdoc/driver/d3d12/d3d12_outputwindow.cpp +++ b/renderdoc/driver/d3d12/d3d12_outputwindow.cpp @@ -26,16 +26,38 @@ #include "d3d12_debug.h" #include "d3d12_device.h" -void D3D12Replay::OutputWindow::MakeRTV(bool multisampled) +void D3D12Replay::OutputWindow::MakeRTV(bool msaa) { SAFE_RELEASE(col); SAFE_RELEASE(colResolve); - D3D12_RESOURCE_DESC texDesc = bb[0]->GetDesc(); + D3D12_RESOURCE_DESC texDesc = {}; + + if(bb[0]) + { + texDesc = bb[0]->GetDesc(); + + texDesc.SampleDesc.Count = msaa ? D3D12_MSAA_SAMPLECOUNT : 1; + + multisampled = msaa; + } + else + { + texDesc.DepthOrArraySize = 1; + texDesc.Dimension = D3D12_RESOURCE_DIMENSION_TEXTURE2D; + texDesc.Flags = D3D12_RESOURCE_FLAG_ALLOW_RENDER_TARGET; + texDesc.Height = height; + texDesc.Layout = D3D12_TEXTURE_LAYOUT_UNKNOWN; + texDesc.MipLevels = 1; + texDesc.SampleDesc.Count = 1; + texDesc.SampleDesc.Quality = 0; + texDesc.Width = width; + + multisampled = false; + } texDesc.Alignment = 0; texDesc.Format = DXGI_FORMAT_R8G8B8A8_UNORM_SRGB; - texDesc.SampleDesc.Count = multisampled ? D3D12_MSAA_SAMPLECOUNT : 1; texDesc.Flags = D3D12_RESOURCE_FLAG_ALLOW_RENDER_TARGET; D3D12_HEAP_PROPERTIES heapProps; @@ -61,7 +83,7 @@ void D3D12Replay::OutputWindow::MakeRTV(bool multisampled) colResolve = NULL; - if(multisampled) + if(msaa) { texDesc.SampleDesc.Count = 1; @@ -97,10 +119,9 @@ void D3D12Replay::OutputWindow::MakeDSV() { SAFE_RELEASE(depth); - D3D12_RESOURCE_DESC texDesc = bb[0]->GetDesc(); + D3D12_RESOURCE_DESC texDesc = col->GetDesc(); texDesc.Alignment = 0; - texDesc.SampleDesc.Count = D3D12_MSAA_SAMPLECOUNT; texDesc.Format = DXGI_FORMAT_D32_FLOAT; texDesc.Flags = D3D12_RESOURCE_FLAG_ALLOW_DEPTH_STENCIL; @@ -140,42 +161,55 @@ void D3D12Replay::OutputWindow::MakeDSV() uint64_t D3D12Replay::MakeOutputWindow(WindowingData window, bool depth) { - RDCASSERT(window.system == WindowingSystem::Win32, window.system); + RDCASSERT(window.system == WindowingSystem::Win32 || window.system == WindowingSystem::Headless, + window.system); - OutputWindow outw; - outw.wnd = window.win32.window; + OutputWindow outw = {}; outw.dev = m_pDevice; - DXGI_SWAP_CHAIN_DESC swapDesc; - RDCEraseEl(swapDesc); - - RECT rect; - GetClientRect(outw.wnd, &rect); - - swapDesc.BufferCount = 2; - swapDesc.BufferDesc.Format = DXGI_FORMAT_R8G8B8A8_UNORM; - outw.width = swapDesc.BufferDesc.Width = rect.right - rect.left; - outw.height = swapDesc.BufferDesc.Height = rect.bottom - rect.top; - swapDesc.BufferUsage = DXGI_USAGE_RENDER_TARGET_OUTPUT; - swapDesc.SampleDesc.Count = 1; - swapDesc.SampleDesc.Quality = 0; - swapDesc.OutputWindow = outw.wnd; - swapDesc.Windowed = TRUE; - swapDesc.SwapEffect = DXGI_SWAP_EFFECT_FLIP_DISCARD; - swapDesc.Flags = 0; - - HRESULT hr = S_OK; - - hr = m_pFactory->CreateSwapChain(m_pDevice->GetQueue(), &swapDesc, &outw.swap); - - if(FAILED(hr)) + if(window.system == WindowingSystem::Win32) { - RDCERR("Failed to create swap chain for HWND, HRESULT: %s", ToStr(hr).c_str()); - return 0; - } + outw.wnd = window.win32.window; - outw.swap->GetBuffer(0, __uuidof(ID3D12Resource), (void **)&outw.bb[0]); - outw.swap->GetBuffer(1, __uuidof(ID3D12Resource), (void **)&outw.bb[1]); + DXGI_SWAP_CHAIN_DESC swapDesc; + RDCEraseEl(swapDesc); + + RECT rect; + GetClientRect(outw.wnd, &rect); + + swapDesc.BufferCount = 2; + swapDesc.BufferDesc.Format = DXGI_FORMAT_R8G8B8A8_UNORM; + outw.width = swapDesc.BufferDesc.Width = rect.right - rect.left; + outw.height = swapDesc.BufferDesc.Height = rect.bottom - rect.top; + swapDesc.BufferUsage = DXGI_USAGE_RENDER_TARGET_OUTPUT; + swapDesc.SampleDesc.Count = 1; + swapDesc.SampleDesc.Quality = 0; + swapDesc.OutputWindow = outw.wnd; + swapDesc.Windowed = TRUE; + swapDesc.SwapEffect = DXGI_SWAP_EFFECT_FLIP_DISCARD; + swapDesc.Flags = 0; + + HRESULT hr = S_OK; + + hr = m_pFactory->CreateSwapChain(m_pDevice->GetQueue(), &swapDesc, &outw.swap); + + if(FAILED(hr)) + { + RDCERR("Failed to create swap chain for HWND, HRESULT: %s", ToStr(hr).c_str()); + return 0; + } + + outw.swap->GetBuffer(0, __uuidof(ID3D12Resource), (void **)&outw.bb[0]); + outw.swap->GetBuffer(1, __uuidof(ID3D12Resource), (void **)&outw.bb[1]); + } + else + { + outw.width = window.headless.width; + outw.height = window.headless.height; + + outw.wnd = NULL; + outw.swap = NULL; + } outw.bbIdx = 0; @@ -187,8 +221,7 @@ uint64_t D3D12Replay::MakeOutputWindow(WindowingData window, bool depth) outw.col = NULL; outw.colResolve = NULL; - outw.MakeRTV(depth); - m_pDevice->CreateRenderTargetView(outw.col, NULL, outw.rtv); + outw.MakeRTV(depth && window.system == WindowingSystem::Win32); outw.depth = NULL; if(depth) @@ -293,6 +326,145 @@ void D3D12Replay::GetOutputWindowDimensions(uint64_t id, int32_t &w, int32_t &h) h = m_OutputWindows[id].height; } +void D3D12Replay::SetOutputWindowDimensions(uint64_t id, int32_t w, int32_t h) +{ + if(id == 0 || m_OutputWindows.find(id) == m_OutputWindows.end()) + return; + + OutputWindow &outw = m_OutputWindows[id]; + + // can't resize an output with an actual window backing + if(outw.wnd) + return; + + m_pDevice->ExecuteLists(); + m_pDevice->FlushLists(true); + + outw.width = w; + outw.height = h; + + outw.MakeRTV(false); + outw.MakeDSV(); + + outw.bbIdx = 0; +} + +void D3D12Replay::GetOutputWindowData(uint64_t id, bytebuf &retData) +{ + if(id == 0 || m_OutputWindows.find(id) == m_OutputWindows.end()) + return; + + OutputWindow &outw = m_OutputWindows[id]; + + if(outw.col == NULL) + return; + + D3D12_HEAP_PROPERTIES heapProps; + heapProps.Type = D3D12_HEAP_TYPE_READBACK; + heapProps.CPUPageProperty = D3D12_CPU_PAGE_PROPERTY_UNKNOWN; + heapProps.MemoryPoolPreference = D3D12_MEMORY_POOL_UNKNOWN; + heapProps.CreationNodeMask = 1; + heapProps.VisibleNodeMask = 1; + + D3D12_RESOURCE_DESC bufDesc; + bufDesc.Alignment = 0; + bufDesc.DepthOrArraySize = 1; + bufDesc.Dimension = D3D12_RESOURCE_DIMENSION_BUFFER; + bufDesc.Flags = D3D12_RESOURCE_FLAG_NONE; + bufDesc.Format = DXGI_FORMAT_UNKNOWN; + bufDesc.Height = 1; + bufDesc.Layout = D3D12_TEXTURE_LAYOUT_ROW_MAJOR; + bufDesc.MipLevels = 1; + bufDesc.SampleDesc.Count = 1; + bufDesc.SampleDesc.Quality = 0; + bufDesc.Width = 1; + + D3D12_RESOURCE_DESC desc = outw.col->GetDesc(); + + D3D12_PLACED_SUBRESOURCE_FOOTPRINT layout = {}; + + m_pDevice->GetCopyableFootprints(&desc, 0, 1, 0, &layout, NULL, NULL, &bufDesc.Width); + + ID3D12Resource *readback = NULL; + HRESULT hr = m_pDevice->CreateCommittedResource(&heapProps, D3D12_HEAP_FLAG_NONE, &bufDesc, + D3D12_RESOURCE_STATE_COPY_DEST, NULL, + __uuidof(ID3D12Resource), (void **)&readback); + + if(SUCCEEDED(hr)) + { + ID3D12GraphicsCommandList *list = m_pDevice->GetNewList(); + + D3D12_RESOURCE_BARRIER barrier = {}; + + // we know there's only one subresource, and it will be in RENDER_TARGET state + barrier.Type = D3D12_RESOURCE_BARRIER_TYPE_TRANSITION; + barrier.Flags = D3D12_RESOURCE_BARRIER_FLAG_NONE; + barrier.Transition.pResource = outw.col; + barrier.Transition.Subresource = 0; + barrier.Transition.StateBefore = D3D12_RESOURCE_STATE_RENDER_TARGET; + barrier.Transition.StateAfter = D3D12_RESOURCE_STATE_COMMON; + + list->ResourceBarrier(1, &barrier); + + // copy to readback buffer + D3D12_TEXTURE_COPY_LOCATION dst, src; + + src.Type = D3D12_TEXTURE_COPY_TYPE_SUBRESOURCE_INDEX; + src.pResource = outw.col; + src.SubresourceIndex = 0; + + dst.Type = D3D12_TEXTURE_COPY_TYPE_PLACED_FOOTPRINT; + dst.pResource = readback; + dst.PlacedFootprint = layout; + + list->CopyTextureRegion(&dst, 0, 0, 0, &src, NULL); + + // transition back + std::swap(barrier.Transition.StateBefore, barrier.Transition.StateAfter); + list->ResourceBarrier(1, &barrier); + + list->Close(); + + m_pDevice->ExecuteLists(NULL, true); + m_pDevice->FlushLists(); + + byte *data = NULL; + hr = readback->Map(0, NULL, (void **)&data); + + if(SUCCEEDED(hr) && data) + { + retData.resize(outw.width * outw.height * 3); + + byte *dstData = retData.data(); + + for(int32_t row = 0; row < outw.height; row++) + { + for(int32_t x = 0; x < outw.width; x++) + { + dstData[x * 3 + 0] = data[x * 4 + 0]; + dstData[x * 3 + 1] = data[x * 4 + 1]; + dstData[x * 3 + 2] = data[x * 4 + 2]; + } + + data += layout.Footprint.RowPitch; + dstData += outw.width * 3; + } + + readback->Unmap(0, NULL); + } + else + { + RDCERR("Couldn't map readback buffer: HRESULT: %s", ToStr(hr).c_str()); + } + + SAFE_RELEASE(readback); + } + else + { + RDCERR("Couldn't create readback buffer: HRESULT: %s", ToStr(hr).c_str()); + } +} + void D3D12Replay::ClearOutputWindowColor(uint64_t id, FloatVector col) { if(id == 0 || m_OutputWindows.find(id) == m_OutputWindows.end()) @@ -328,7 +500,7 @@ void D3D12Replay::BindOutputWindow(uint64_t id, bool depth) m_CurrentOutputWindow = id; - if(outw.bb[0] == NULL) + if(outw.col == NULL) return; SetOutputDimensions(outw.width, outw.height); @@ -339,6 +511,9 @@ bool D3D12Replay::IsOutputWindowVisible(uint64_t id) if(id == 0 || m_OutputWindows.find(id) == m_OutputWindows.end()) return false; + if(!m_OutputWindows[id].wnd) + return true; + return (IsWindowVisible(m_OutputWindows[id].wnd) == TRUE); } @@ -349,7 +524,7 @@ void D3D12Replay::FlipOutputWindow(uint64_t id) OutputWindow &outw = m_OutputWindows[id]; - if(m_OutputWindows[id].bb[0] == NULL) + if(m_OutputWindows[id].bb[0] == NULL || m_OutputWindows[id].swap == NULL) return; D3D12_RESOURCE_BARRIER barriers[3]; @@ -358,7 +533,7 @@ void D3D12Replay::FlipOutputWindow(uint64_t id) barriers[0].Transition.pResource = outw.col; barriers[0].Transition.StateBefore = D3D12_RESOURCE_STATE_RENDER_TARGET; barriers[0].Transition.StateAfter = - outw.depth ? D3D12_RESOURCE_STATE_RESOLVE_SOURCE : D3D12_RESOURCE_STATE_COPY_SOURCE; + outw.multisampled ? D3D12_RESOURCE_STATE_RESOLVE_SOURCE : D3D12_RESOURCE_STATE_COPY_SOURCE; barriers[1].Transition.pResource = outw.bb[outw.bbIdx]; barriers[1].Transition.StateBefore = D3D12_RESOURCE_STATE_PRESENT; @@ -371,7 +546,7 @@ void D3D12Replay::FlipOutputWindow(uint64_t id) ID3D12GraphicsCommandList *list = m_pDevice->GetNewList(); // resolve or copy from colour to backbuffer - if(outw.depth) + if(outw.multisampled) { // transition colour to resolve source, resolve target to resolve dest, backbuffer to copy dest list->ResourceBarrier(3, barriers); diff --git a/renderdoc/driver/d3d12/d3d12_replay.cpp b/renderdoc/driver/d3d12/d3d12_replay.cpp index 386657ac3..fdcd93860 100644 --- a/renderdoc/driver/d3d12/d3d12_replay.cpp +++ b/renderdoc/driver/d3d12/d3d12_replay.cpp @@ -1639,7 +1639,8 @@ void D3D12Replay::RenderCheckerboard() list->IASetPrimitiveTopology(D3D_PRIMITIVE_TOPOLOGY_TRIANGLESTRIP); - list->SetPipelineState(outw.depth ? m_General.CheckerboardMSAAPipe : m_General.CheckerboardPipe); + list->SetPipelineState(outw.multisampled ? m_General.CheckerboardMSAAPipe + : m_General.CheckerboardPipe); list->SetGraphicsRootSignature(m_General.CheckerboardRootSig); diff --git a/renderdoc/driver/d3d12/d3d12_replay.h b/renderdoc/driver/d3d12/d3d12_replay.h index 9c7d86a11..4365aeeaa 100644 --- a/renderdoc/driver/d3d12/d3d12_replay.h +++ b/renderdoc/driver/d3d12/d3d12_replay.h @@ -113,6 +113,8 @@ public: void DestroyOutputWindow(uint64_t id); bool CheckResizeOutputWindow(uint64_t id); void GetOutputWindowDimensions(uint64_t id, int32_t &w, int32_t &h); + void SetOutputWindowDimensions(uint64_t id, int32_t w, int32_t h); + void GetOutputWindowData(uint64_t id, bytebuf &retData); void ClearOutputWindowColor(uint64_t id, FloatVector col); void ClearOutputWindowDepth(uint64_t id, float depth, uint8_t stencil); void BindOutputWindow(uint64_t id, bool depth); @@ -299,6 +301,7 @@ private: void MakeDSV(); int width, height; + bool multisampled; }; float m_OutputWidth = 1.0f; diff --git a/renderdoc/driver/gl/cgl_platform.cpp b/renderdoc/driver/gl/cgl_platform.cpp index f0bc559ec..d6ea47338 100644 --- a/renderdoc/driver/gl/cgl_platform.cpp +++ b/renderdoc/driver/gl/cgl_platform.cpp @@ -198,7 +198,7 @@ class CGLPlatform : public GLPlatform return ret; } - else if(window.system == WindowingSystem::Unknown) + else if(window.system == WindowingSystem::Unknown || window.system == WindowingSystem::Headless) { ret.nsctx = NSGL_createContext(NULL, share_context.nsctx); diff --git a/renderdoc/driver/gl/egl_platform.cpp b/renderdoc/driver/gl/egl_platform.cpp index e960f9fc6..176c35445 100644 --- a/renderdoc/driver/gl/egl_platform.cpp +++ b/renderdoc/driver/gl/egl_platform.cpp @@ -125,7 +125,8 @@ class EGLPlatform : public GLPlatform case WindowingSystem::Xlib: win = window.xlib.window; break; #endif case WindowingSystem::Unknown: - // allow WindowingSystem::Unknown so that internally we can create a window-less context + case WindowingSystem::Headless: + // allow Unknown and Headless so that internally we can create a window-less context break; default: RDCERR("Unexpected window system %u", window.system); break; } diff --git a/renderdoc/driver/gl/gl_outputwindow.cpp b/renderdoc/driver/gl/gl_outputwindow.cpp index 84cf647d1..29a469ce5 100644 --- a/renderdoc/driver/gl/gl_outputwindow.cpp +++ b/renderdoc/driver/gl/gl_outputwindow.cpp @@ -100,7 +100,7 @@ bool GLReplay::CheckResizeOutputWindow(uint64_t id) OutputWindow &outw = m_OutputWindows[id]; - if(outw.ctx == 0) + if(outw.ctx == NULL || outw.system == WindowingSystem::Headless) return false; int32_t w, h; @@ -184,6 +184,9 @@ void GLReplay::FlipOutputWindow(uint64_t id) OutputWindow &outw = m_OutputWindows[id]; + if(outw.system == WindowingSystem::Headless) + return; + MakeCurrentReplayContext(&outw); WrappedOpenGL &drv = *m_pDriver; @@ -213,7 +216,17 @@ uint64_t GLReplay::MakeOutputWindow(WindowingData window, bool depth) if(!win.ctx) return 0; - m_pDriver->m_Platform.GetOutputWindowDimensions(win, win.width, win.height); + win.system = window.system; + + if(window.system == WindowingSystem::Headless) + { + win.width = window.headless.width; + win.height = window.headless.height; + } + else + { + m_pDriver->m_Platform.GetOutputWindowDimensions(win, win.width, win.height); + } MakeCurrentReplayContext(&win); @@ -253,13 +266,106 @@ void GLReplay::GetOutputWindowDimensions(uint64_t id, int32_t &w, int32_t &h) OutputWindow &outw = m_OutputWindows[id]; + if(outw.system == WindowingSystem::Headless) + { + w = outw.width; + h = outw.height; + return; + } + m_pDriver->m_Platform.GetOutputWindowDimensions(outw, w, h); } +void GLReplay::SetOutputWindowDimensions(uint64_t id, int32_t w, int32_t h) +{ + if(id == 0 || m_OutputWindows.find(id) == m_OutputWindows.end()) + return; + + OutputWindow &outw = m_OutputWindows[id]; + + // can't resize an output with an actual window backing + if(outw.system != WindowingSystem::Headless) + return; + + outw.width = w; + outw.height = h; + + MakeCurrentReplayContext(m_DebugCtx); + + WrappedOpenGL &drv = *m_pDriver; + + bool haddepth = false; + + drv.glDeleteTextures(1, &outw.BlitData.backbuffer); + if(outw.BlitData.depthstencil) + { + haddepth = true; + drv.glDeleteTextures(1, &outw.BlitData.depthstencil); + } + drv.glDeleteFramebuffers(1, &outw.BlitData.windowFBO); + + CreateOutputWindowBackbuffer(outw, haddepth); +} + +void GLReplay::GetOutputWindowData(uint64_t id, bytebuf &retData) +{ + if(id == 0 || m_OutputWindows.find(id) == m_OutputWindows.end()) + return; + + OutputWindow &outw = m_OutputWindows[id]; + + MakeCurrentReplayContext(m_DebugCtx); + + m_pDriver->glBindFramebuffer(eGL_READ_FRAMEBUFFER, outw.BlitData.windowFBO); + m_pDriver->glReadBuffer(eGL_BACK); + m_pDriver->glBindBuffer(eGL_PIXEL_PACK_BUFFER, 0); + m_pDriver->glPixelStorei(eGL_PACK_ROW_LENGTH, 0); + m_pDriver->glPixelStorei(eGL_PACK_SKIP_ROWS, 0); + m_pDriver->glPixelStorei(eGL_PACK_SKIP_PIXELS, 0); + m_pDriver->glPixelStorei(eGL_PACK_ALIGNMENT, 1); + + // read as RGBA for maximum compatibility. + retData.resize(outw.width * outw.height * 4); + GL.glReadPixels(0, 0, outw.width, outw.height, eGL_RGBA, eGL_UNSIGNED_BYTE, retData.data()); + + // y-flip + for(int32_t row = 0; row < outw.height / 2; row++) + { + const uint32_t stride = outw.width * 4; + const int32_t fliprow = outw.height - 1 - row; + + for(int32_t x = 0; x < outw.width; x++) + { + std::swap(retData[row * stride + x * 4 + 0], retData[fliprow * stride + x * 4 + 0]); + std::swap(retData[row * stride + x * 4 + 1], retData[fliprow * stride + x * 4 + 1]); + std::swap(retData[row * stride + x * 4 + 2], retData[fliprow * stride + x * 4 + 2]); + std::swap(retData[row * stride + x * 4 + 3], retData[fliprow * stride + x * 4 + 3]); + } + } + + // compact from RGBA to RGB. + byte *src = retData.data(); + byte *dst = retData.data(); + for(int32_t row = 0; row < outw.height; row++) + { + for(int32_t x = 0; x < outw.width; x++) + { + memcpy(dst, src, 3); + dst += 3; + src += 4; + } + } + + retData.resize(outw.width * outw.height * 3); +} + bool GLReplay::IsOutputWindowVisible(uint64_t id) { if(id == 0 || m_OutputWindows.find(id) == m_OutputWindows.end()) return false; + if(m_OutputWindows[id].system == WindowingSystem::Headless) + return true; + return m_pDriver->m_Platform.IsOutputWindowVisible(m_OutputWindows[id]); } diff --git a/renderdoc/driver/gl/gl_replay.h b/renderdoc/driver/gl/gl_replay.h index 066af254b..05d3cd83c 100644 --- a/renderdoc/driver/gl/gl_replay.h +++ b/renderdoc/driver/gl/gl_replay.h @@ -137,6 +137,8 @@ public: void DestroyOutputWindow(uint64_t id); bool CheckResizeOutputWindow(uint64_t id); void GetOutputWindowDimensions(uint64_t id, int32_t &w, int32_t &h); + void SetOutputWindowDimensions(uint64_t id, int32_t w, int32_t h); + void GetOutputWindowData(uint64_t id, bytebuf &retData); void ClearOutputWindowColor(uint64_t id, FloatVector col); void ClearOutputWindowDepth(uint64_t id, float depth, uint8_t stencil); void BindOutputWindow(uint64_t id, bool depth); @@ -279,6 +281,7 @@ private: GLuint readFBO = 0; } BlitData; + WindowingSystem system = WindowingSystem::Headless; int width = 1, height = 1; }; diff --git a/renderdoc/driver/gl/glx_platform.cpp b/renderdoc/driver/gl/glx_platform.cpp index c43ff1a91..d11debc43 100644 --- a/renderdoc/driver/gl/glx_platform.cpp +++ b/renderdoc/driver/gl/glx_platform.cpp @@ -151,7 +151,7 @@ class GLXPlatform : public GLPlatform "support compiled in"); #endif } - else if(window.system == WindowingSystem::Unknown) + else if(window.system == WindowingSystem::Unknown || window.system == WindowingSystem::Headless) { // allow WindowingSystem::Unknown so that internally we can create a window-less context dpy = RenderDoc::Inst().GetGlobalEnvironment().xlibDisplay; diff --git a/renderdoc/driver/gl/wgl_platform.cpp b/renderdoc/driver/gl/wgl_platform.cpp index dc5a0d240..453265195 100644 --- a/renderdoc/driver/gl/wgl_platform.cpp +++ b/renderdoc/driver/gl/wgl_platform.cpp @@ -107,12 +107,15 @@ class WGLPlatform : public GLPlatform if(!WGL.wglGetPixelFormatAttribivARB || !WGL.wglCreateContextAttribsARB) return ret; - RDCASSERT(window.system == WindowingSystem::Win32 || window.system == WindowingSystem::Unknown, + RDCASSERT(window.system == WindowingSystem::Win32 || window.system == WindowingSystem::Unknown || + window.system == WindowingSystem::Headless, window.system); - HWND w = window.win32.window; + HWND w = NULL; - if(w == NULL) + if(window.system == WindowingSystem::Win32) + w = window.win32.window; + else w = CreateWindowEx(WS_EX_CLIENTEDGE, WINDOW_CLASS_NAME, L"", WS_OVERLAPPEDWINDOW, CW_USEDEFAULT, CW_USEDEFAULT, CW_USEDEFAULT, CW_USEDEFAULT, NULL, NULL, GetModuleHandle(NULL), NULL); diff --git a/renderdoc/driver/vulkan/vk_android.cpp b/renderdoc/driver/vulkan/vk_android.cpp index 89acf39bd..f8f2f581a 100644 --- a/renderdoc/driver/vulkan/vk_android.cpp +++ b/renderdoc/driver/vulkan/vk_android.cpp @@ -77,6 +77,13 @@ void VulkanReplay::GetOutputWindowDimensions(uint64_t id, int32_t &w, int32_t &h OutputWindow &outw = m_OutputWindows[id]; + if(outw.m_WindowSystem == WindowingSystem::Headless) + { + w = outw.width; + h = outw.height; + return; + } + w = ANativeWindow_getWidth(outw.wnd); h = ANativeWindow_getHeight(outw.wnd); } diff --git a/renderdoc/driver/vulkan/vk_apple.cpp b/renderdoc/driver/vulkan/vk_apple.cpp index 1bace56ee..173fd0a49 100644 --- a/renderdoc/driver/vulkan/vk_apple.cpp +++ b/renderdoc/driver/vulkan/vk_apple.cpp @@ -83,6 +83,13 @@ void VulkanReplay::GetOutputWindowDimensions(uint64_t id, int32_t &w, int32_t &h OutputWindow &outw = m_OutputWindows[id]; + if(outw.m_WindowSystem == WindowingSystem::Headless) + { + w = outw.width; + h = outw.height; + return; + } + w = getMetalLayerWidth(outw.wnd); h = getMetalLayerHeight(outw.wnd); } diff --git a/renderdoc/driver/vulkan/vk_ggp.cpp b/renderdoc/driver/vulkan/vk_ggp.cpp index 0ae508c13..ceff16f2c 100644 --- a/renderdoc/driver/vulkan/vk_ggp.cpp +++ b/renderdoc/driver/vulkan/vk_ggp.cpp @@ -76,6 +76,13 @@ void VulkanReplay::GetOutputWindowDimensions(uint64_t id, int32_t &w, int32_t &h OutputWindow &outw = m_OutputWindows[id]; + if(outw.m_WindowSystem == WindowingSystem::Headless) + { + w = outw.width; + h = outw.height; + return; + } + RDCLOG("Window system is GGP (%d), size is %d, %d", outw.m_WindowSystem, outw.width, outw.height); // No window, specify default resolution. w = outw.width != 0 ? outw.width : 1920; diff --git a/renderdoc/driver/vulkan/vk_linux.cpp b/renderdoc/driver/vulkan/vk_linux.cpp index e2080dde3..c75899052 100644 --- a/renderdoc/driver/vulkan/vk_linux.cpp +++ b/renderdoc/driver/vulkan/vk_linux.cpp @@ -203,6 +203,13 @@ void VulkanReplay::GetOutputWindowDimensions(uint64_t id, int32_t &w, int32_t &h OutputWindow &outw = m_OutputWindows[id]; + if(outw.m_WindowSystem == WindowingSystem::Headless) + { + w = outw.width; + h = outw.height; + return; + } + #if ENABLED(RDOC_XLIB) if(outw.m_WindowSystem == WindowingSystem::Xlib) { diff --git a/renderdoc/driver/vulkan/vk_outputwindow.cpp b/renderdoc/driver/vulkan/vk_outputwindow.cpp index 4952a13e4..db8935199 100644 --- a/renderdoc/driver/vulkan/vk_outputwindow.cpp +++ b/renderdoc/driver/vulkan/vk_outputwindow.cpp @@ -85,14 +85,6 @@ VulkanReplay::OutputWindow::OutputWindow() VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT; } -void VulkanReplay::OutputWindow::SetCol(VkDeviceMemory mem, VkImage img) -{ -} - -void VulkanReplay::OutputWindow::SetDS(VkDeviceMemory mem, VkImage img) -{ -} - void VulkanReplay::OutputWindow::Destroy(WrappedVulkan *driver, VkDevice device) { const VkLayerDispatchTable *vt = ObjDisp(device); @@ -197,7 +189,7 @@ void VulkanReplay::OutputWindow::Create(WrappedVulkan *driver, VkDevice device, fresh = true; - if(surface == VK_NULL_HANDLE) + if(surface == VK_NULL_HANDLE && m_WindowSystem != WindowingSystem::Headless) { CreateSurface(inst); @@ -211,182 +203,185 @@ void VulkanReplay::OutputWindow::Create(WrappedVulkan *driver, VkDevice device, VkResult vkr = VK_SUCCESS; - VkSurfaceCapabilitiesKHR capabilities; - - ObjDisp(inst)->GetPhysicalDeviceSurfaceCapabilitiesKHR(Unwrap(phys), Unwrap(surface), - &capabilities); - - RDCASSERT(capabilities.supportedUsageFlags & VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT); - // AMD didn't report this capability for a while. If the assert fires for you, update - // your drivers! - RDCASSERT(capabilities.supportedUsageFlags & VK_IMAGE_USAGE_TRANSFER_DST_BIT); - - RDCASSERT(capabilities.minImageCount <= 2 && - (2 <= capabilities.maxImageCount || capabilities.maxImageCount == 0)); - - // check format and present mode from driver + if(m_WindowSystem != WindowingSystem::Headless) { - uint32_t numFormats = 0; + VkSurfaceCapabilitiesKHR capabilities; - vkr = ObjDisp(inst)->GetPhysicalDeviceSurfaceFormatsKHR(Unwrap(phys), Unwrap(surface), - &numFormats, NULL); - RDCASSERTEQUAL(vkr, VK_SUCCESS); + ObjDisp(inst)->GetPhysicalDeviceSurfaceCapabilitiesKHR(Unwrap(phys), Unwrap(surface), + &capabilities); - if(numFormats > 0) + RDCASSERT(capabilities.supportedUsageFlags & VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT); + // AMD didn't report this capability for a while. If the assert fires for you, update + // your drivers! + RDCASSERT(capabilities.supportedUsageFlags & VK_IMAGE_USAGE_TRANSFER_DST_BIT); + + RDCASSERT(capabilities.minImageCount <= 2 && + (2 <= capabilities.maxImageCount || capabilities.maxImageCount == 0)); + + // check format and present mode from driver { - VkSurfaceFormatKHR *formats = new VkSurfaceFormatKHR[numFormats]; + uint32_t numFormats = 0; vkr = ObjDisp(inst)->GetPhysicalDeviceSurfaceFormatsKHR(Unwrap(phys), Unwrap(surface), - &numFormats, formats); + &numFormats, NULL); RDCASSERTEQUAL(vkr, VK_SUCCESS); - if(numFormats == 1 && formats[0].format == VK_FORMAT_UNDEFINED) + if(numFormats > 0) { - // 1 entry with undefined means no preference, just use our default - imformat = VK_FORMAT_B8G8R8A8_SRGB; - imcolspace = VK_COLORSPACE_SRGB_NONLINEAR_KHR; - } - else - { - // try and find a format with SRGB correction - imformat = VK_FORMAT_UNDEFINED; - imcolspace = formats[0].colorSpace; + VkSurfaceFormatKHR *formats = new VkSurfaceFormatKHR[numFormats]; - for(uint32_t i = 0; i < numFormats; i++) + vkr = ObjDisp(inst)->GetPhysicalDeviceSurfaceFormatsKHR(Unwrap(phys), Unwrap(surface), + &numFormats, formats); + RDCASSERTEQUAL(vkr, VK_SUCCESS); + + if(numFormats == 1 && formats[0].format == VK_FORMAT_UNDEFINED) { - if(IsSRGBFormat(formats[i].format)) + // 1 entry with undefined means no preference, just use our default + imformat = VK_FORMAT_B8G8R8A8_SRGB; + imcolspace = VK_COLORSPACE_SRGB_NONLINEAR_KHR; + } + else + { + // try and find a format with SRGB correction + imformat = VK_FORMAT_UNDEFINED; + imcolspace = formats[0].colorSpace; + + for(uint32_t i = 0; i < numFormats; i++) { - imformat = formats[i].format; - imcolspace = formats[i].colorSpace; - RDCASSERT(imcolspace == VK_COLORSPACE_SRGB_NONLINEAR_KHR); - break; + if(IsSRGBFormat(formats[i].format)) + { + imformat = formats[i].format; + imcolspace = formats[i].colorSpace; + RDCASSERT(imcolspace == VK_COLORSPACE_SRGB_NONLINEAR_KHR); + break; + } + } + + if(imformat == VK_FORMAT_UNDEFINED) + { + RDCWARN("Couldn't find SRGB correcting output swapchain format"); + imformat = formats[0].format; } } - if(imformat == VK_FORMAT_UNDEFINED) - { - RDCWARN("Couldn't find SRGB correcting output swapchain format"); - imformat = formats[0].format; - } + SAFE_DELETE_ARRAY(formats); } - SAFE_DELETE_ARRAY(formats); - } - - uint32_t numModes = 0; - - vkr = ObjDisp(inst)->GetPhysicalDeviceSurfacePresentModesKHR(Unwrap(phys), Unwrap(surface), - &numModes, NULL); - RDCASSERTEQUAL(vkr, VK_SUCCESS); - - if(numModes > 0) - { - VkPresentModeKHR *modes = new VkPresentModeKHR[numModes]; + uint32_t numModes = 0; vkr = ObjDisp(inst)->GetPhysicalDeviceSurfacePresentModesKHR(Unwrap(phys), Unwrap(surface), - &numModes, modes); + &numModes, NULL); RDCASSERTEQUAL(vkr, VK_SUCCESS); - // If mailbox mode is available, use it, as is the lowest-latency non- - // tearing mode. If not, try IMMEDIATE which will usually be available, - // and is fastest (though it tears). If not, fall back to FIFO which is - // always available. - for(size_t i = 0; i < numModes; i++) + if(numModes > 0) { - if(modes[i] == VK_PRESENT_MODE_MAILBOX_KHR) + VkPresentModeKHR *modes = new VkPresentModeKHR[numModes]; + + vkr = ObjDisp(inst)->GetPhysicalDeviceSurfacePresentModesKHR(Unwrap(phys), Unwrap(surface), + &numModes, modes); + RDCASSERTEQUAL(vkr, VK_SUCCESS); + + // If mailbox mode is available, use it, as is the lowest-latency non- + // tearing mode. If not, try IMMEDIATE which will usually be available, + // and is fastest (though it tears). If not, fall back to FIFO which is + // always available. + for(size_t i = 0; i < numModes; i++) { - presentmode = VK_PRESENT_MODE_MAILBOX_KHR; - break; + if(modes[i] == VK_PRESENT_MODE_MAILBOX_KHR) + { + presentmode = VK_PRESENT_MODE_MAILBOX_KHR; + break; + } + + if(modes[i] == VK_PRESENT_MODE_IMMEDIATE_KHR) + presentmode = VK_PRESENT_MODE_IMMEDIATE_KHR; } - if(modes[i] == VK_PRESENT_MODE_IMMEDIATE_KHR) - presentmode = VK_PRESENT_MODE_IMMEDIATE_KHR; + SAFE_DELETE_ARRAY(modes); } - - SAFE_DELETE_ARRAY(modes); } + + VkBool32 supported = false; + ObjDisp(inst)->GetPhysicalDeviceSurfaceSupportKHR(Unwrap(phys), driver->GetQFamilyIdx(), + Unwrap(surface), &supported); + + // can't really recover from this anyway + RDCASSERT(supported); + + VkSwapchainCreateInfoKHR swapInfo = { + VK_STRUCTURE_TYPE_SWAPCHAIN_CREATE_INFO_KHR, + NULL, + 0, + Unwrap(surface), + 2, + imformat, + imcolspace, + {width, height}, + 1, + VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT, + VK_SHARING_MODE_EXCLUSIVE, + 0, + NULL, + VK_SURFACE_TRANSFORM_IDENTITY_BIT_KHR, + VK_COMPOSITE_ALPHA_OPAQUE_BIT_KHR, + presentmode, + true, + Unwrap(old), + }; + + vkr = vt->CreateSwapchainKHR(Unwrap(device), &swapInfo, NULL, &swap); + RDCASSERTEQUAL(vkr, VK_SUCCESS); + + if(old != VK_NULL_HANDLE) + { + vt->DestroySwapchainKHR(Unwrap(device), Unwrap(old), NULL); + GetResourceManager()->ReleaseWrappedResource(old); + } + + if(swap == VK_NULL_HANDLE) + { + RDCERR("Failed to create swapchain. %d consecutive failures!", failures); + failures++; + + // do some sort of backoff. + + // the first time, try to recreate again next frame + if(failures == 1) + recreatePause = 0; + // the next few times, wait 200 'frames' between attempts + else if(failures < 10) + recreatePause = 100; + // otherwise, only reattempt very infrequently. A resize will + // always retrigger a recreate, so ew probably don't want to + // try again + else + recreatePause = 1000; + + return; + } + + failures = 0; + + GetResourceManager()->WrapResource(Unwrap(device), swap); + + vkr = vt->GetSwapchainImagesKHR(Unwrap(device), Unwrap(swap), &numImgs, NULL); + RDCASSERTEQUAL(vkr, VK_SUCCESS); + + VkImage *imgs = new VkImage[numImgs]; + vkr = vt->GetSwapchainImagesKHR(Unwrap(device), Unwrap(swap), &numImgs, imgs); + RDCASSERTEQUAL(vkr, VK_SUCCESS); + + for(size_t i = 0; i < numImgs; i++) + { + colimg[i] = imgs[i]; + GetResourceManager()->WrapResource(Unwrap(device), colimg[i]); + colBarrier[i].image = Unwrap(colimg[i]); + colBarrier[i].oldLayout = colBarrier[i].newLayout = VK_IMAGE_LAYOUT_UNDEFINED; + } + + delete[] imgs; } - VkBool32 supported = false; - ObjDisp(inst)->GetPhysicalDeviceSurfaceSupportKHR(Unwrap(phys), driver->GetQFamilyIdx(), - Unwrap(surface), &supported); - - // can't really recover from this anyway - RDCASSERT(supported); - - VkSwapchainCreateInfoKHR swapInfo = { - VK_STRUCTURE_TYPE_SWAPCHAIN_CREATE_INFO_KHR, - NULL, - 0, - Unwrap(surface), - 2, - imformat, - imcolspace, - {width, height}, - 1, - VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT, - VK_SHARING_MODE_EXCLUSIVE, - 0, - NULL, - VK_SURFACE_TRANSFORM_IDENTITY_BIT_KHR, - VK_COMPOSITE_ALPHA_OPAQUE_BIT_KHR, - presentmode, - true, - Unwrap(old), - }; - - vkr = vt->CreateSwapchainKHR(Unwrap(device), &swapInfo, NULL, &swap); - RDCASSERTEQUAL(vkr, VK_SUCCESS); - - if(old != VK_NULL_HANDLE) - { - vt->DestroySwapchainKHR(Unwrap(device), Unwrap(old), NULL); - GetResourceManager()->ReleaseWrappedResource(old); - } - - if(swap == VK_NULL_HANDLE) - { - RDCERR("Failed to create swapchain. %d consecutive failures!", failures); - failures++; - - // do some sort of backoff. - - // the first time, try to recreate again next frame - if(failures == 1) - recreatePause = 0; - // the next few times, wait 200 'frames' between attempts - else if(failures < 10) - recreatePause = 100; - // otherwise, only reattempt very infrequently. A resize will - // always retrigger a recreate, so ew probably don't want to - // try again - else - recreatePause = 1000; - - return; - } - - failures = 0; - - GetResourceManager()->WrapResource(Unwrap(device), swap); - - vkr = vt->GetSwapchainImagesKHR(Unwrap(device), Unwrap(swap), &numImgs, NULL); - RDCASSERTEQUAL(vkr, VK_SUCCESS); - - VkImage *imgs = new VkImage[numImgs]; - vkr = vt->GetSwapchainImagesKHR(Unwrap(device), Unwrap(swap), &numImgs, imgs); - RDCASSERTEQUAL(vkr, VK_SUCCESS); - - for(size_t i = 0; i < numImgs; i++) - { - colimg[i] = imgs[i]; - GetResourceManager()->WrapResource(Unwrap(device), colimg[i]); - colBarrier[i].image = Unwrap(colimg[i]); - colBarrier[i].oldLayout = colBarrier[i].newLayout = VK_IMAGE_LAYOUT_UNDEFINED; - } - - delete[] imgs; - curidx = 0; // for our 'fake' backbuffer, create in RGBA8 @@ -639,6 +634,143 @@ void VulkanReplay::OutputWindow::Create(WrappedVulkan *driver, VkDevice device, } } +void VulkanReplay::GetOutputWindowData(uint64_t id, bytebuf &retData) +{ + if(id == 0 || m_OutputWindows.find(id) == m_OutputWindows.end()) + return; + + OutputWindow &outw = m_OutputWindows[id]; + + VkDevice device = m_pDriver->GetDev(); + VkCommandBuffer cmd = m_pDriver->GetNextCmd(); + + const VkLayerDispatchTable *vt = ObjDisp(device); + + vt->DeviceWaitIdle(Unwrap(device)); + + VkBuffer readbackBuf = VK_NULL_HANDLE; + + VkResult vkr = VK_SUCCESS; + + // create readback buffer + VkBufferCreateInfo bufInfo = { + VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, + NULL, + 0, + GetByteSize(outw.width, outw.height, 1, VK_FORMAT_R8G8B8A8_UNORM, 0), + VK_BUFFER_USAGE_TRANSFER_DST_BIT, + }; + vt->CreateBuffer(Unwrap(device), &bufInfo, NULL, &readbackBuf); + RDCASSERTEQUAL(vkr, VK_SUCCESS); + + VkMemoryRequirements mrq = {0}; + + vt->GetBufferMemoryRequirements(Unwrap(device), readbackBuf, &mrq); + + VkMemoryAllocateInfo allocInfo = { + VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, NULL, bufInfo.size, + m_pDriver->GetReadbackMemoryIndex(mrq.memoryTypeBits), + }; + + VkDeviceMemory readbackMem = VK_NULL_HANDLE; + vkr = vt->AllocateMemory(Unwrap(device), &allocInfo, NULL, &readbackMem); + RDCASSERTEQUAL(vkr, VK_SUCCESS); + + vkr = vt->BindBufferMemory(Unwrap(device), readbackBuf, readbackMem, 0); + RDCASSERTEQUAL(vkr, VK_SUCCESS); + + VkCommandBufferBeginInfo beginInfo = {VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, NULL, + VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT}; + + // do image copy + vkr = vt->BeginCommandBuffer(Unwrap(cmd), &beginInfo); + RDCASSERTEQUAL(vkr, VK_SUCCESS); + + VkBufferImageCopy cpy = { + 0, + 0, + 0, + {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}, + { + 0, 0, 0, + }, + {outw.width, outw.height, 1}, + }; + + outw.bbBarrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + outw.bbBarrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + + DoPipelineBarrier(cmd, 1, &outw.bbBarrier); + + vt->CmdCopyImageToBuffer(Unwrap(cmd), Unwrap(outw.bb), VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + readbackBuf, 1, &cpy); + + outw.bbBarrier.oldLayout = outw.bbBarrier.newLayout; + outw.bbBarrier.srcAccessMask = outw.bbBarrier.dstAccessMask; + + vkr = vt->EndCommandBuffer(Unwrap(cmd)); + RDCASSERTEQUAL(vkr, VK_SUCCESS); + + m_pDriver->SubmitCmds(); + m_pDriver->FlushQ(); // need to wait so we can readback + + // map memory and readback + byte *pData = NULL; + vkr = vt->MapMemory(Unwrap(device), readbackMem, 0, bufInfo.size, 0, (void **)&pData); + RDCASSERTEQUAL(vkr, VK_SUCCESS); + RDCASSERT(pData != NULL); + + VkMappedMemoryRange range = { + VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE, NULL, readbackMem, 0, bufInfo.size, + }; + + vkr = vt->InvalidateMappedMemoryRanges(Unwrap(device), 1, &range); + RDCASSERTEQUAL(vkr, VK_SUCCESS); + + { + retData.resize(outw.width * outw.height * 3); + + byte *src = (byte *)pData; + byte *dst = retData.data(); + + for(uint32_t row = 0; row < outw.height; row++) + { + for(uint32_t x = 0; x < outw.width; x++) + { + dst[x * 3 + 0] = src[x * 4 + 0]; + dst[x * 3 + 1] = src[x * 4 + 1]; + dst[x * 3 + 2] = src[x * 4 + 2]; + } + + src += outw.width * 4; + dst += outw.width * 3; + } + } + + vt->UnmapMemory(Unwrap(device), readbackMem); + + // delete all + vt->DestroyBuffer(Unwrap(device), readbackBuf, NULL); + vt->FreeMemory(Unwrap(device), readbackMem, NULL); +} + +void VulkanReplay::SetOutputWindowDimensions(uint64_t id, int32_t w, int32_t h) +{ + if(id == 0 || m_OutputWindows.find(id) == m_OutputWindows.end()) + return; + + OutputWindow &outw = m_OutputWindows[id]; + + // can't resize an output with an actual window backing + if(outw.m_WindowSystem != WindowingSystem::Headless) + return; + + outw.width = w; + outw.height = h; + + outw.Create(m_pDriver, m_pDriver->GetDev(), outw.hasDepth); +} + bool VulkanReplay::CheckResizeOutputWindow(uint64_t id) { if(id == 0 || m_OutputWindows.find(id) == m_OutputWindows.end()) @@ -646,7 +778,8 @@ bool VulkanReplay::CheckResizeOutputWindow(uint64_t id) OutputWindow &outw = m_OutputWindows[id]; - if(outw.m_WindowSystem == WindowingSystem::Unknown) + if(outw.m_WindowSystem == WindowingSystem::Unknown || + outw.m_WindowSystem == WindowingSystem::Headless) return false; int32_t w, h; @@ -687,7 +820,7 @@ void VulkanReplay::BindOutputWindow(uint64_t id, bool depth) // if the swapchain failed to create, do nothing. We will try to recreate it // again in CheckResizeOutputWindow (once per render 'frame') - if(outw.swap == VK_NULL_HANDLE) + if(outw.m_WindowSystem != WindowingSystem::Headless && outw.swap == VK_NULL_HANDLE) return; m_DebugWidth = (int32_t)outw.width; @@ -696,53 +829,58 @@ void VulkanReplay::BindOutputWindow(uint64_t id, bool depth) VkDevice dev = m_pDriver->GetDev(); VkCommandBuffer cmd = m_pDriver->GetNextCmd(); const VkLayerDispatchTable *vt = ObjDisp(dev); + VkResult vkr = VK_SUCCESS; - // semaphore is short lived, so not wrapped, if it's cached (ideally) - // then it should be wrapped - VkSemaphore sem; - VkPipelineStageFlags stage = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT; - VkSemaphoreCreateInfo semInfo = {VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO, NULL, 0}; - - VkResult vkr = vt->CreateSemaphore(Unwrap(dev), &semInfo, NULL, &sem); - RDCASSERTEQUAL(vkr, VK_SUCCESS); - - vkr = vt->AcquireNextImageKHR(Unwrap(dev), Unwrap(outw.swap), UINT64_MAX, sem, VK_NULL_HANDLE, - &outw.curidx); - - if(vkr == VK_ERROR_OUT_OF_DATE_KHR) + // if we have a swapchain, acquire the next image. + if(outw.swap != VK_NULL_HANDLE) { - // force a swapchain recreate. - outw.width = 0; - outw.height = 0; + // semaphore is short lived, so not wrapped, if it's cached (ideally) + // then it should be wrapped + VkSemaphore sem; + VkPipelineStageFlags stage = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT; + VkSemaphoreCreateInfo semInfo = {VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO, NULL, 0}; - CheckResizeOutputWindow(id); + vkr = vt->CreateSemaphore(Unwrap(dev), &semInfo, NULL, &sem); + RDCASSERTEQUAL(vkr, VK_SUCCESS); - // then try again to acquire. vkr = vt->AcquireNextImageKHR(Unwrap(dev), Unwrap(outw.swap), UINT64_MAX, sem, VK_NULL_HANDLE, &outw.curidx); + + if(vkr == VK_ERROR_OUT_OF_DATE_KHR) + { + // force a swapchain recreate. + outw.width = 0; + outw.height = 0; + + CheckResizeOutputWindow(id); + + // then try again to acquire. + vkr = vt->AcquireNextImageKHR(Unwrap(dev), Unwrap(outw.swap), UINT64_MAX, sem, VK_NULL_HANDLE, + &outw.curidx); + } + + RDCASSERTEQUAL(vkr, VK_SUCCESS); + + VkSubmitInfo submitInfo = { + VK_STRUCTURE_TYPE_SUBMIT_INFO, + NULL, + 1, + &sem, + &stage, + 0, + NULL, // cmd buffers + 0, + NULL, // signal semaphores + }; + + vkr = vt->QueueSubmit(Unwrap(m_pDriver->GetQ()), 1, &submitInfo, VK_NULL_HANDLE); + RDCASSERTEQUAL(vkr, VK_SUCCESS); + + vt->QueueWaitIdle(Unwrap(m_pDriver->GetQ())); + + vt->DestroySemaphore(Unwrap(dev), sem, NULL); } - RDCASSERTEQUAL(vkr, VK_SUCCESS); - - VkSubmitInfo submitInfo = { - VK_STRUCTURE_TYPE_SUBMIT_INFO, - NULL, - 1, - &sem, - &stage, - 0, - NULL, // cmd buffers - 0, - NULL, // signal semaphores - }; - - vkr = vt->QueueSubmit(Unwrap(m_pDriver->GetQ()), 1, &submitInfo, VK_NULL_HANDLE); - RDCASSERTEQUAL(vkr, VK_SUCCESS); - - vt->QueueWaitIdle(Unwrap(m_pDriver->GetQ())); - - vt->DestroySemaphore(Unwrap(dev), sem, NULL); - VkCommandBufferBeginInfo beginInfo = {VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, NULL, VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT}; @@ -775,7 +913,8 @@ void VulkanReplay::BindOutputWindow(uint64_t id, bool depth) outw.colBarrier[outw.curidx].dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; DoPipelineBarrier(cmd, 1, &outw.bbBarrier); - DoPipelineBarrier(cmd, 1, &outw.colBarrier[outw.curidx]); + if(outw.colimg[0] != VK_NULL_HANDLE) + DoPipelineBarrier(cmd, 1, &outw.colBarrier[outw.curidx]); if(outw.dsimg != VK_NULL_HANDLE) DoPipelineBarrier(cmd, 1, &outw.depthBarrier); @@ -802,7 +941,7 @@ void VulkanReplay::ClearOutputWindowColor(uint64_t id, FloatVector col) // if the swapchain failed to create, do nothing. We will try to recreate it // again in CheckResizeOutputWindow (once per render 'frame') - if(outw.swap == VK_NULL_HANDLE) + if(outw.m_WindowSystem != WindowingSystem::Headless && outw.swap == VK_NULL_HANDLE) return; VkDevice dev = m_pDriver->GetDev(); @@ -852,7 +991,7 @@ void VulkanReplay::ClearOutputWindowDepth(uint64_t id, float depth, uint8_t sten // if the swapchain failed to create, do nothing. We will try to recreate it // again in CheckResizeOutputWindow (once per render 'frame') - if(outw.swap == VK_NULL_HANDLE) + if(outw.m_WindowSystem != WindowingSystem::Headless && outw.swap == VK_NULL_HANDLE) return; VkDevice dev = m_pDriver->GetDev(); @@ -1059,13 +1198,24 @@ uint64_t VulkanReplay::MakeOutputWindow(WindowingData window, bool depth) m_OutputWinID++; m_OutputWindows[id].m_WindowSystem = window.system; - m_OutputWindows[id].SetWindowHandle(window); m_OutputWindows[id].m_ResourceManager = GetResourceManager(); + if(window.system != WindowingSystem::Unknown && window.system != WindowingSystem::Headless) + m_OutputWindows[id].SetWindowHandle(window); + if(window.system != WindowingSystem::Unknown) { int32_t w, h; - GetOutputWindowDimensions(id, w, h); + + if(window.system == WindowingSystem::Headless) + { + w = window.headless.width; + h = window.headless.height; + } + else + { + GetOutputWindowDimensions(id, w, h); + } m_OutputWindows[id].width = w; m_OutputWindows[id].height = h; diff --git a/renderdoc/driver/vulkan/vk_posix.cpp b/renderdoc/driver/vulkan/vk_posix.cpp index e97eec0f6..75694a1af 100644 --- a/renderdoc/driver/vulkan/vk_posix.cpp +++ b/renderdoc/driver/vulkan/vk_posix.cpp @@ -36,6 +36,9 @@ bool VulkanReplay::IsOutputWindowVisible(uint64_t id) if(id == 0 || m_OutputWindows.find(id) == m_OutputWindows.end()) return false; + if(m_OutputWindows[id].m_WindowSystem == WindowingSystem::Headless) + return true; + VULKANNOTIMP("Optimisation missing - output window always returning true"); return true; diff --git a/renderdoc/driver/vulkan/vk_rendermesh.cpp b/renderdoc/driver/vulkan/vk_rendermesh.cpp index e4e426df2..f0f50b7bc 100644 --- a/renderdoc/driver/vulkan/vk_rendermesh.cpp +++ b/renderdoc/driver/vulkan/vk_rendermesh.cpp @@ -370,7 +370,7 @@ void VulkanReplay::RenderMesh(uint32_t eventId, const vector &second // if the swapchain failed to create, do nothing. We will try to recreate it // again in CheckResizeOutputWindow (once per render 'frame') - if(outw.swap == VK_NULL_HANDLE) + if(outw.m_WindowSystem != WindowingSystem::Headless && outw.swap == VK_NULL_HANDLE) return; VkDevice dev = m_pDriver->GetDev(); diff --git a/renderdoc/driver/vulkan/vk_rendertexture.cpp b/renderdoc/driver/vulkan/vk_rendertexture.cpp index 39dfc3129..c75104991 100644 --- a/renderdoc/driver/vulkan/vk_rendertexture.cpp +++ b/renderdoc/driver/vulkan/vk_rendertexture.cpp @@ -129,7 +129,7 @@ bool VulkanReplay::RenderTexture(TextureDisplay cfg) // if the swapchain failed to create, do nothing. We will try to recreate it // again in CheckResizeOutputWindow (once per render 'frame') - if(outw.swap == VK_NULL_HANDLE) + if(outw.m_WindowSystem != WindowingSystem::Headless && outw.swap == VK_NULL_HANDLE) return false; VkRenderPassBeginInfo rpbegin = { diff --git a/renderdoc/driver/vulkan/vk_replay.cpp b/renderdoc/driver/vulkan/vk_replay.cpp index 62bc64374..5689bac56 100644 --- a/renderdoc/driver/vulkan/vk_replay.cpp +++ b/renderdoc/driver/vulkan/vk_replay.cpp @@ -580,7 +580,7 @@ void VulkanReplay::RenderCheckerboard() // if the swapchain failed to create, do nothing. We will try to recreate it // again in CheckResizeOutputWindow (once per render 'frame') - if(outw.swap == VK_NULL_HANDLE) + if(outw.m_WindowSystem != WindowingSystem::Headless && outw.swap == VK_NULL_HANDLE) return; VkDevice dev = m_pDriver->GetDev(); @@ -706,7 +706,7 @@ void VulkanReplay::RenderHighlightBox(float w, float h, float scale) // if the swapchain failed to create, do nothing. We will try to recreate it // again in CheckResizeOutputWindow (once per render 'frame') - if(outw.swap == VK_NULL_HANDLE) + if(outw.m_WindowSystem != WindowingSystem::Headless && outw.swap == VK_NULL_HANDLE) return; VkDevice dev = m_pDriver->GetDev(); diff --git a/renderdoc/driver/vulkan/vk_replay.h b/renderdoc/driver/vulkan/vk_replay.h index 7c83bdd2a..67f5bd353 100644 --- a/renderdoc/driver/vulkan/vk_replay.h +++ b/renderdoc/driver/vulkan/vk_replay.h @@ -274,6 +274,8 @@ public: void DestroyOutputWindow(uint64_t id); bool CheckResizeOutputWindow(uint64_t id); void GetOutputWindowDimensions(uint64_t id, int32_t &w, int32_t &h); + void SetOutputWindowDimensions(uint64_t id, int32_t w, int32_t h); + void GetOutputWindowData(uint64_t id, bytebuf &retData); void ClearOutputWindowColor(uint64_t id, FloatVector col); void ClearOutputWindowDepth(uint64_t id, float depth, uint8_t stencil); void BindOutputWindow(uint64_t id, bool depth); @@ -403,8 +405,6 @@ private: { OutputWindow(); - void SetCol(VkDeviceMemory mem, VkImage img); - void SetDS(VkDeviceMemory mem, VkImage img); void Create(WrappedVulkan *driver, VkDevice device, bool depth); void Destroy(WrappedVulkan *driver, VkDevice device); diff --git a/renderdoc/driver/vulkan/vk_win32.cpp b/renderdoc/driver/vulkan/vk_win32.cpp index 4e1e33c05..90a86c65e 100644 --- a/renderdoc/driver/vulkan/vk_win32.cpp +++ b/renderdoc/driver/vulkan/vk_win32.cpp @@ -60,6 +60,13 @@ void VulkanReplay::GetOutputWindowDimensions(uint64_t id, int32_t &w, int32_t &h OutputWindow &outw = m_OutputWindows[id]; + if(outw.m_WindowSystem == WindowingSystem::Headless) + { + w = outw.width; + h = outw.height; + return; + } + RECT rect = {0}; GetClientRect(outw.wnd, &rect); w = rect.right - rect.left; @@ -71,6 +78,9 @@ bool VulkanReplay::IsOutputWindowVisible(uint64_t id) if(id == 0 || m_OutputWindows.find(id) == m_OutputWindows.end()) return false; + if(m_OutputWindows[id].m_WindowSystem == WindowingSystem::Headless) + return true; + return (IsWindowVisible(m_OutputWindows[id].wnd) == TRUE); } diff --git a/renderdoc/replay/replay_controller.h b/renderdoc/replay/replay_controller.h index 5c7b54437..f2cc7ec9f 100644 --- a/renderdoc/replay/replay_controller.h +++ b/renderdoc/replay/replay_controller.h @@ -43,6 +43,8 @@ public: void SetTextureDisplay(const TextureDisplay &o); void SetMeshDisplay(const MeshDisplay &o); void SetDimensions(int32_t width, int32_t height); + bytebuf ReadbackOutputTexture(); + rdcpair GetDimensions(); void ClearThumbnails(); bool AddThumbnail(WindowingData window, ResourceId texID, CompType typeHint); diff --git a/renderdoc/replay/replay_driver.h b/renderdoc/replay/replay_driver.h index c3023fcf6..83ed1507c 100644 --- a/renderdoc/replay/replay_driver.h +++ b/renderdoc/replay/replay_driver.h @@ -186,7 +186,9 @@ public: virtual uint64_t MakeOutputWindow(WindowingData window, bool depth) = 0; virtual void DestroyOutputWindow(uint64_t id) = 0; virtual bool CheckResizeOutputWindow(uint64_t id) = 0; + virtual void SetOutputWindowDimensions(uint64_t id, int32_t w, int32_t h) = 0; virtual void GetOutputWindowDimensions(uint64_t id, int32_t &w, int32_t &h) = 0; + virtual void GetOutputWindowData(uint64_t id, bytebuf &retData) = 0; virtual void ClearOutputWindowColor(uint64_t id, FloatVector col) = 0; virtual void ClearOutputWindowDepth(uint64_t id, float depth, uint8_t stencil) = 0; virtual void BindOutputWindow(uint64_t id, bool depth) = 0; diff --git a/renderdoc/replay/replay_output.cpp b/renderdoc/replay/replay_output.cpp index 5e0ace053..a26fcfc73 100644 --- a/renderdoc/replay/replay_output.cpp +++ b/renderdoc/replay/replay_output.cpp @@ -116,17 +116,6 @@ ReplayOutput::ReplayOutput(ReplayController *parent, WindowingData window, Repla RenderDoc::Inst().GetCrashHandler()->RegisterMemoryRegion(this, sizeof(ReplayController)); } -void ReplayOutput::SetDimensions(int32_t width, int32_t height) -{ - CHECK_REPLAY_THREAD(); - - if(m_MainOutput.outputID == 0) - { - m_Width = width; - m_Height = height; - } -} - ReplayOutput::~ReplayOutput() { CHECK_REPLAY_THREAD(); @@ -146,6 +135,31 @@ void ReplayOutput::Shutdown() m_pRenderer->ShutdownOutput(this); } +void ReplayOutput::SetDimensions(int32_t width, int32_t height) +{ + CHECK_REPLAY_THREAD(); + + m_pDevice->SetOutputWindowDimensions(m_MainOutput.outputID, width > 0 ? width : 1, + height > 0 ? height : 1); + m_pDevice->GetOutputWindowDimensions(m_MainOutput.outputID, m_Width, m_Height); +} + +bytebuf ReplayOutput::ReadbackOutputTexture() +{ + CHECK_REPLAY_THREAD(); + + bytebuf data; + m_pDevice->GetOutputWindowData(m_MainOutput.outputID, data); + return data; +} + +rdcpair ReplayOutput::GetDimensions() +{ + CHECK_REPLAY_THREAD(); + + return make_rdcpair(m_Width, m_Height); +} + void ReplayOutput::SetTextureDisplay(const TextureDisplay &o) { CHECK_REPLAY_THREAD(); @@ -300,7 +314,7 @@ bool ReplayOutput::AddThumbnail(WindowingData window, ResourceId texID, CompType OutputPair p; - RDCASSERT(window.system != WindowingSystem::Unknown); + RDCASSERT(window.system != WindowingSystem::Unknown && window.system != WindowingSystem::Headless); bool depthMode = false; diff --git a/util/test/tests/D3D11/D3D11_Overlay_Test.py b/util/test/tests/D3D11/D3D11_Overlay_Test.py index 5f22d792d..6be579ae4 100644 --- a/util/test/tests/D3D11/D3D11_Overlay_Test.py +++ b/util/test/tests/D3D11/D3D11_Overlay_Test.py @@ -11,12 +11,10 @@ class D3D11_Overlay_Test(rdtest.TestCase): def check_capture(self): self.check_final_backbuffer() - out: rd.ReplayOutput = self.controller.CreateOutput(rd.CreateHeadlessWindowingData(), rd.ReplayOutputType.Texture) + out: rd.ReplayOutput = self.controller.CreateOutput(rd.CreateHeadlessWindowingData(100, 100), rd.ReplayOutputType.Texture) self.check(out is not None) - out.SetDimensions(100, 100) - test_marker: rd.DrawcallDescription = self.find_draw("Test") self.controller.SetFrameEvent(test_marker.next.eventId, True) diff --git a/util/test/tests/D3D12/D3D12_Overlay_Test.py b/util/test/tests/D3D12/D3D12_Overlay_Test.py index 9e7c87dd5..f0e3968ab 100644 --- a/util/test/tests/D3D12/D3D12_Overlay_Test.py +++ b/util/test/tests/D3D12/D3D12_Overlay_Test.py @@ -12,12 +12,10 @@ class D3D12_Overlay_Test(rdtest.TestCase): def check_capture(self): self.check_final_backbuffer() - out: rd.ReplayOutput = self.controller.CreateOutput(rd.CreateHeadlessWindowingData(), rd.ReplayOutputType.Texture) + out: rd.ReplayOutput = self.controller.CreateOutput(rd.CreateHeadlessWindowingData(100, 100), rd.ReplayOutputType.Texture) self.check(out is not None) - out.SetDimensions(100, 100) - test_marker: rd.DrawcallDescription = self.find_draw("Test") self.controller.SetFrameEvent(test_marker.next.eventId, True) diff --git a/util/test/tests/GL/GL_Overlay_Test.py b/util/test/tests/GL/GL_Overlay_Test.py index a3d880a8b..0c9ba034c 100644 --- a/util/test/tests/GL/GL_Overlay_Test.py +++ b/util/test/tests/GL/GL_Overlay_Test.py @@ -9,12 +9,10 @@ class GL_Overlay_Test(rdtest.TestCase): def check_capture(self): self.check_final_backbuffer() - out: rd.ReplayOutput = self.controller.CreateOutput(rd.CreateHeadlessWindowingData(), rd.ReplayOutputType.Texture) + out: rd.ReplayOutput = self.controller.CreateOutput(rd.CreateHeadlessWindowingData(100, 100), rd.ReplayOutputType.Texture) self.check(out is not None) - out.SetDimensions(100, 100) - test_marker: rd.DrawcallDescription = self.find_draw("Test") self.controller.SetFrameEvent(test_marker.next.eventId, True) diff --git a/util/test/tests/Vulkan/VK_Overlay_Test.py b/util/test/tests/Vulkan/VK_Overlay_Test.py index 0c190f24a..ac4008a49 100644 --- a/util/test/tests/Vulkan/VK_Overlay_Test.py +++ b/util/test/tests/Vulkan/VK_Overlay_Test.py @@ -9,12 +9,10 @@ class VK_Overlay_Test(rdtest.TestCase): def check_capture(self): self.check_final_backbuffer() - out: rd.ReplayOutput = self.controller.CreateOutput(rd.CreateHeadlessWindowingData(), rd.ReplayOutputType.Texture) + out: rd.ReplayOutput = self.controller.CreateOutput(rd.CreateHeadlessWindowingData(100, 100), rd.ReplayOutputType.Texture) self.check(out is not None) - out.SetDimensions(100, 100) - test_marker: rd.DrawcallDescription = self.find_draw("Test") self.controller.SetFrameEvent(test_marker.next.eventId, True)