Allow creating completely headless outputs and reading back their output

* This is useful when writing automated tests that want to test the output of
  rendering which only happens to outputs, such as mesh rendering, or could
  potentially be bypassed with direct readback like GetTextureData vs rendering
  a texture.

renderdoc/api/replay/renderdoc_replay.h

@@ -310,7 +310,12 @@ DOCUMENT(R"(Specifies a windowing system to use for creating an output window.
 
 .. data:: Unknown
 
-  No windowing data is passed and no native window is described.
+  Unknown window type, no windowing data is passed and no native window is described.
+
+.. data:: Headless
+
+  The windowing data doesn't describe a real window but a virtual area, allowing all normal output
+  rendering to happen off-screen.
   See :func:`CreateHeadlessWindowingData`.
 
 .. data:: Win32
@@ -337,6 +342,7 @@ DOCUMENT(R"(Specifies a windowing system to use for creating an output window.
 enum class WindowingSystem : uint32_t
 {
   Unknown,
+  Headless,
   Win32,
   Xlib,
   XCB,
@@ -381,6 +387,11 @@ struct WindowingData
 
   union
   {
+    struct
+    {
+      int32_t width, height;
+    } headless;
+
     struct
     {
       HWND window;
@@ -417,14 +428,20 @@ DECLARE_REFLECTION_ENUM(WindowingData);
 
 DOCUMENT(R"(Create a :class:`WindowingData` for no backing window, it will be headless.
 
+:param int width: The initial width for this virtual window.
+:param int height: The initial height for this virtual window.
+
 :return: A :class:`WindowingData` corresponding to an 'empty' backing window.
 :rtype: WindowingData
 )");
-inline const WindowingData CreateHeadlessWindowingData()
+inline const WindowingData CreateHeadlessWindowingData(int32_t width, int32_t height)
 {
   WindowingData ret = {};
 
-  ret.system = WindowingSystem::Unknown;
+  ret.system = WindowingSystem::Headless;
+
+  ret.headless.width = width > 0 ? width : 1;
+  ret.headless.height = height > 0 ? height : 1;
 
   return ret;
 }
@@ -649,6 +666,21 @@ which is useful for operations like picking vertices that depends on the output
 )");
   virtual void SetDimensions(int32_t width, int32_t height) = 0;
 
+  DOCUMENT(R"(Read the output texture back as byte data. Primarily useful for headless outputs where
+the output data is not displayed anywhere natively.
+
+:return: The output texture data as tightly packed RGB 3-byte data.
+:rtype: ``bytes``
+)");
+  virtual bytebuf ReadbackOutputTexture() = 0;
+
+  DOCUMENT(R"(Retrieve the current dimensions of the output.
+
+:return: The current width and height of the output.
+:rtype: ``pair`` of two ``int``
+)");
+  virtual rdcpair<int32_t, int32_t> GetDimensions() = 0;
+
   DOCUMENT(
       "Clear and release all thumbnails associated with this output. See :meth:`AddThumbnail`.");
   virtual void ClearThumbnails() = 0;

renderdoc/core/image_viewer.cpp

@@ -81,10 +81,18 @@ public:
   }
   void DestroyOutputWindow(uint64_t id) { m_Proxy->DestroyOutputWindow(id); }
   bool CheckResizeOutputWindow(uint64_t id) { return m_Proxy->CheckResizeOutputWindow(id); }
+  void SetOutputWindowDimensions(uint64_t id, int32_t w, int32_t h)
+  {
+    m_Proxy->SetOutputWindowDimensions(id, w, h);
+  }
   void GetOutputWindowDimensions(uint64_t id, int32_t &w, int32_t &h)
   {
     m_Proxy->GetOutputWindowDimensions(id, w, h);
   }
+  void GetOutputWindowData(uint64_t id, bytebuf &retData)
+  {
+    m_Proxy->GetOutputWindowData(id, retData);
+  }
   void ClearOutputWindowColor(uint64_t id, FloatVector col)
   {
     m_Proxy->ClearOutputWindowColor(id, col);

renderdoc/core/replay_proxy.h

@@ -195,6 +195,16 @@ public:
     if(m_Proxy)
       return m_Proxy->GetOutputWindowDimensions(id, w, h);
   }
+  void SetOutputWindowDimensions(uint64_t id, int32_t w, int32_t h)
+  {
+    if(m_Proxy)
+      m_Proxy->SetOutputWindowDimensions(id, w, h);
+  }
+  void GetOutputWindowData(uint64_t id, bytebuf &retData)
+  {
+    if(m_Proxy)
+      m_Proxy->GetOutputWindowData(id, retData);
+  }
   void ClearOutputWindowColor(uint64_t id, FloatVector col)
   {
     if(m_Proxy)

renderdoc/driver/d3d11/d3d11_outputwindow.cpp

@@ -29,13 +29,37 @@
 void D3D11Replay::OutputWindow::MakeRTV()
 {
   ID3D11Texture2D *texture = NULL;
-  HRESULT hr = swap->GetBuffer(0, __uuidof(ID3D11Texture2D), (void **)&texture);
 
-  if(FAILED(hr))
+  HRESULT hr = S_OK;
+
+  if(swap)
   {
-    RDCERR("Failed to get swap chain buffer, HRESULT: %s", ToStr(hr).c_str());
-    SAFE_RELEASE(texture);
-    return;
+    hr = swap->GetBuffer(0, __uuidof(ID3D11Texture2D), (void **)&texture);
+
+    if(FAILED(hr))
+    {
+      RDCERR("Failed to get swap chain buffer, HRESULT: %s", ToStr(hr).c_str());
+      SAFE_RELEASE(texture);
+      return;
+    }
+  }
+  else
+  {
+    D3D11_TEXTURE2D_DESC texDesc;
+
+    texDesc.ArraySize = 1;
+    texDesc.BindFlags = D3D11_BIND_SHADER_RESOURCE | D3D11_BIND_RENDER_TARGET;
+    texDesc.CPUAccessFlags = 0;
+    texDesc.MipLevels = 1;
+    texDesc.MiscFlags = 0;
+    texDesc.SampleDesc.Count = 1;
+    texDesc.SampleDesc.Quality = 0;
+    texDesc.Usage = D3D11_USAGE_DEFAULT;
+    texDesc.Width = width;
+    texDesc.Height = height;
+    texDesc.Format = DXGI_FORMAT_R8G8B8A8_UNORM_SRGB;
+
+    hr = dev->CreateTexture2D(&texDesc, NULL, &texture);
   }
 
   hr = dev->CreateRenderTargetView(texture, NULL, &rtv);
@@ -52,13 +76,19 @@ void D3D11Replay::OutputWindow::MakeRTV()
 
 void D3D11Replay::OutputWindow::MakeDSV()
 {
-  ID3D11Texture2D *texture = NULL;
-  HRESULT hr = swap->GetBuffer(0, __uuidof(ID3D11Texture2D), (void **)&texture);
+  if(!rtv)
+    return;
 
-  if(FAILED(hr))
+  ID3D11Texture2D *texture = NULL;
   {
-    RDCERR("Failed to get swap chain buffer, HRESULT: %s", ToStr(hr).c_str());
-    SAFE_RELEASE(texture);
+    ID3D11Resource *res = NULL;
+    rtv->GetResource(&res);
+    texture = (ID3D11Texture2D *)res;
+  }
+
+  if(!texture)
+  {
+    RDCERR("Failed to get swap chain buffer from RTV");
     return;
   }
 
@@ -70,7 +100,7 @@ void D3D11Replay::OutputWindow::MakeDSV()
   texDesc.Format = DXGI_FORMAT_D24_UNORM_S8_UINT;
   texDesc.BindFlags = D3D11_BIND_DEPTH_STENCIL;
 
-  hr = dev->CreateTexture2D(&texDesc, NULL, &texture);
+  HRESULT hr = dev->CreateTexture2D(&texDesc, NULL, &texture);
 
   if(FAILED(hr))
   {
@@ -95,38 +125,49 @@ void D3D11Replay::OutputWindow::MakeDSV()
 
 uint64_t D3D11Replay::MakeOutputWindow(WindowingData window, bool depth)
 {
-  RDCASSERT(window.system == WindowingSystem::Win32, window.system);
+  RDCASSERT(window.system == WindowingSystem::Win32 || window.system == WindowingSystem::Headless,
+            window.system);
 
-  OutputWindow outw;
-  outw.wnd = window.win32.window;
+  DXGI_SWAP_CHAIN_DESC swapDesc = {};
+  OutputWindow outw = {};
   outw.dev = m_pDevice;
 
-  DXGI_SWAP_CHAIN_DESC swapDesc;
-  RDCEraseEl(swapDesc);
-
-  RECT rect;
-  GetClientRect(outw.wnd, &rect);
-
-  swapDesc.BufferCount = 2;
-  swapDesc.BufferDesc.Format = DXGI_FORMAT_R8G8B8A8_UNORM_SRGB;
-  outw.width = swapDesc.BufferDesc.Width = rect.right - rect.left;
-  outw.height = swapDesc.BufferDesc.Height = rect.bottom - rect.top;
-  swapDesc.BufferUsage = DXGI_USAGE_RENDER_TARGET_OUTPUT;
-  swapDesc.SampleDesc.Count = depth ? 4 : 1;
-  swapDesc.SampleDesc.Quality = 0;
-  swapDesc.OutputWindow = outw.wnd;
-  swapDesc.Windowed = TRUE;
-  swapDesc.SwapEffect = DXGI_SWAP_EFFECT_DISCARD;
-  swapDesc.Flags = 0;
-
-  HRESULT hr = S_OK;
-
-  hr = m_pFactory->CreateSwapChain(m_pDevice, &swapDesc, &outw.swap);
-
-  if(FAILED(hr))
+  if(window.system == WindowingSystem::Win32)
   {
-    RDCERR("Failed to create swap chain for HWND, HRESULT: %s", ToStr(hr).c_str());
-    return 0;
+    outw.wnd = window.win32.window;
+
+    RECT rect = {};
+    GetClientRect(outw.wnd, &rect);
+
+    swapDesc.BufferCount = 2;
+    swapDesc.BufferDesc.Format = DXGI_FORMAT_R8G8B8A8_UNORM_SRGB;
+    outw.width = swapDesc.BufferDesc.Width = rect.right - rect.left;
+    outw.height = swapDesc.BufferDesc.Height = rect.bottom - rect.top;
+    swapDesc.BufferUsage = DXGI_USAGE_RENDER_TARGET_OUTPUT;
+    swapDesc.SampleDesc.Count = depth ? 4 : 1;
+    swapDesc.SampleDesc.Quality = 0;
+    swapDesc.OutputWindow = outw.wnd;
+    swapDesc.Windowed = TRUE;
+    swapDesc.SwapEffect = DXGI_SWAP_EFFECT_DISCARD;
+    swapDesc.Flags = 0;
+
+    HRESULT hr = S_OK;
+
+    hr = m_pFactory->CreateSwapChain(m_pDevice, &swapDesc, &outw.swap);
+
+    if(FAILED(hr))
+    {
+      RDCERR("Failed to create swap chain for HWND, HRESULT: %s", ToStr(hr).c_str());
+      return 0;
+    }
+  }
+  else
+  {
+    outw.width = window.headless.width;
+    outw.height = window.headless.height;
+
+    outw.wnd = NULL;
+    outw.swap = NULL;
   }
 
   outw.MakeRTV();
@@ -215,6 +256,100 @@ void D3D11Replay::GetOutputWindowDimensions(uint64_t id, int32_t &w, int32_t &h)
   h = m_OutputWindows[id].height;
 }
 
+void D3D11Replay::SetOutputWindowDimensions(uint64_t id, int32_t w, int32_t h)
+{
+  if(id == 0 || m_OutputWindows.find(id) == m_OutputWindows.end())
+    return;
+
+  OutputWindow &outw = m_OutputWindows[id];
+
+  // can't resize an output with an actual window backing
+  if(outw.wnd)
+    return;
+
+  SAFE_RELEASE(outw.rtv);
+  SAFE_RELEASE(outw.dsv);
+
+  outw.width = w;
+  outw.height = h;
+
+  outw.MakeRTV();
+  outw.MakeDSV();
+}
+
+void D3D11Replay::GetOutputWindowData(uint64_t id, bytebuf &retData)
+{
+  if(id == 0 || m_OutputWindows.find(id) == m_OutputWindows.end())
+    return;
+
+  OutputWindow &outw = m_OutputWindows[id];
+
+  if(!outw.rtv)
+    return;
+
+  ID3D11Texture2D *texture = NULL;
+  {
+    ID3D11Resource *res = NULL;
+    outw.rtv->GetResource(&res);
+    texture = (ID3D11Texture2D *)res;
+  }
+
+  if(!texture)
+  {
+    RDCERR("Couldn't get backbuffer texture");
+    return;
+  }
+
+  ID3D11Texture2D *readback = NULL;
+
+  D3D11_TEXTURE2D_DESC texDesc;
+  texture->GetDesc(&texDesc);
+
+  texDesc.CPUAccessFlags = D3D11_CPU_ACCESS_READ;
+  texDesc.BindFlags = 0;
+  texDesc.Usage = D3D11_USAGE_STAGING;
+
+  HRESULT hr = m_pDevice->CreateTexture2D(&texDesc, NULL, &readback);
+
+  if(FAILED(hr))
+  {
+    RDCERR("Couldn't create staging texture for readback, HRESULT: %s", ToStr(hr).c_str());
+    SAFE_RELEASE(texture);
+    return;
+  }
+
+  ID3D11DeviceContext *ctx = m_pDevice->GetImmediateContext();
+
+  ctx->CopyResource(readback, texture);
+
+  SAFE_RELEASE(texture);
+
+  D3D11_MAPPED_SUBRESOURCE mapped = {};
+  ctx->Map(readback, 0, D3D11_MAP_READ, 0, &mapped);
+
+  retData.resize(outw.width * outw.height * 3);
+
+  byte *src = (byte *)mapped.pData;
+  byte *dst = retData.data();
+
+  for(int32_t row = 0; row < outw.height; row++)
+  {
+    for(int32_t x = 0; x < outw.width; x++)
+    {
+      dst[x * 3 + 0] = src[x * 4 + 0];
+      dst[x * 3 + 1] = src[x * 4 + 1];
+      dst[x * 3 + 2] = src[x * 4 + 2];
+    }
+
+    src += mapped.RowPitch;
+    dst += outw.width * 3;
+  }
+
+  ctx->Unmap(readback, 0);
+
+  SAFE_RELEASE(readback);
+}
+
 void D3D11Replay::ClearOutputWindowColor(uint64_t id, FloatVector col)
 {
   if(id == 0 || m_OutputWindows.find(id) == m_OutputWindows.end())
@@ -259,6 +394,9 @@ bool D3D11Replay::IsOutputWindowVisible(uint64_t id)
   if(id == 0 || m_OutputWindows.find(id) == m_OutputWindows.end())
     return false;
 
+  if(!m_OutputWindows[id].wnd)
+    return true;
+
   return (IsWindowVisible(m_OutputWindows[id].wnd) == TRUE);
 }
 

renderdoc/driver/d3d11/d3d11_replay.h

@@ -154,6 +154,8 @@ public:
   void DestroyOutputWindow(uint64_t id);
   bool CheckResizeOutputWindow(uint64_t id);
   void GetOutputWindowDimensions(uint64_t id, int32_t &w, int32_t &h);
+  void SetOutputWindowDimensions(uint64_t id, int32_t w, int32_t h);
+  void GetOutputWindowData(uint64_t id, bytebuf &retData);
   void ClearOutputWindowColor(uint64_t id, FloatVector col);
   void ClearOutputWindowDepth(uint64_t id, float depth, uint8_t stencil);
   void BindOutputWindow(uint64_t id, bool depth);

renderdoc/driver/d3d12/d3d12_outputwindow.cpp

@@ -26,16 +26,38 @@
 #include "d3d12_debug.h"
 #include "d3d12_device.h"
 
-void D3D12Replay::OutputWindow::MakeRTV(bool multisampled)
+void D3D12Replay::OutputWindow::MakeRTV(bool msaa)
 {
   SAFE_RELEASE(col);
   SAFE_RELEASE(colResolve);
 
-  D3D12_RESOURCE_DESC texDesc = bb[0]->GetDesc();
+  D3D12_RESOURCE_DESC texDesc = {};
+
+  if(bb[0])
+  {
+    texDesc = bb[0]->GetDesc();
+
+    texDesc.SampleDesc.Count = msaa ? D3D12_MSAA_SAMPLECOUNT : 1;
+
+    multisampled = msaa;
+  }
+  else
+  {
+    texDesc.DepthOrArraySize = 1;
+    texDesc.Dimension = D3D12_RESOURCE_DIMENSION_TEXTURE2D;
+    texDesc.Flags = D3D12_RESOURCE_FLAG_ALLOW_RENDER_TARGET;
+    texDesc.Height = height;
+    texDesc.Layout = D3D12_TEXTURE_LAYOUT_UNKNOWN;
+    texDesc.MipLevels = 1;
+    texDesc.SampleDesc.Count = 1;
+    texDesc.SampleDesc.Quality = 0;
+    texDesc.Width = width;
+
+    multisampled = false;
+  }
 
   texDesc.Alignment = 0;
   texDesc.Format = DXGI_FORMAT_R8G8B8A8_UNORM_SRGB;
-  texDesc.SampleDesc.Count = multisampled ? D3D12_MSAA_SAMPLECOUNT : 1;
   texDesc.Flags = D3D12_RESOURCE_FLAG_ALLOW_RENDER_TARGET;
 
   D3D12_HEAP_PROPERTIES heapProps;
@@ -61,7 +83,7 @@ void D3D12Replay::OutputWindow::MakeRTV(bool multisampled)
 
   colResolve = NULL;
 
-  if(multisampled)
+  if(msaa)
   {
     texDesc.SampleDesc.Count = 1;
 
@@ -97,10 +119,9 @@ void D3D12Replay::OutputWindow::MakeDSV()
 {
   SAFE_RELEASE(depth);
 
-  D3D12_RESOURCE_DESC texDesc = bb[0]->GetDesc();
+  D3D12_RESOURCE_DESC texDesc = col->GetDesc();
 
   texDesc.Alignment = 0;
-  texDesc.SampleDesc.Count = D3D12_MSAA_SAMPLECOUNT;
   texDesc.Format = DXGI_FORMAT_D32_FLOAT;
   texDesc.Flags = D3D12_RESOURCE_FLAG_ALLOW_DEPTH_STENCIL;
 
@@ -140,42 +161,55 @@ void D3D12Replay::OutputWindow::MakeDSV()
 
 uint64_t D3D12Replay::MakeOutputWindow(WindowingData window, bool depth)
 {
-  RDCASSERT(window.system == WindowingSystem::Win32, window.system);
+  RDCASSERT(window.system == WindowingSystem::Win32 || window.system == WindowingSystem::Headless,
+            window.system);
 
-  OutputWindow outw;
-  outw.wnd = window.win32.window;
+  OutputWindow outw = {};
   outw.dev = m_pDevice;
 
-  DXGI_SWAP_CHAIN_DESC swapDesc;
-  RDCEraseEl(swapDesc);
-
-  RECT rect;
-  GetClientRect(outw.wnd, &rect);
-
-  swapDesc.BufferCount = 2;
-  swapDesc.BufferDesc.Format = DXGI_FORMAT_R8G8B8A8_UNORM;
-  outw.width = swapDesc.BufferDesc.Width = rect.right - rect.left;
-  outw.height = swapDesc.BufferDesc.Height = rect.bottom - rect.top;
-  swapDesc.BufferUsage = DXGI_USAGE_RENDER_TARGET_OUTPUT;
-  swapDesc.SampleDesc.Count = 1;
-  swapDesc.SampleDesc.Quality = 0;
-  swapDesc.OutputWindow = outw.wnd;
-  swapDesc.Windowed = TRUE;
-  swapDesc.SwapEffect = DXGI_SWAP_EFFECT_FLIP_DISCARD;
-  swapDesc.Flags = 0;
-
-  HRESULT hr = S_OK;
-
-  hr = m_pFactory->CreateSwapChain(m_pDevice->GetQueue(), &swapDesc, &outw.swap);
-
-  if(FAILED(hr))
+  if(window.system == WindowingSystem::Win32)
   {
-    RDCERR("Failed to create swap chain for HWND, HRESULT: %s", ToStr(hr).c_str());
-    return 0;
-  }
+    outw.wnd = window.win32.window;
 
-  outw.swap->GetBuffer(0, __uuidof(ID3D12Resource), (void **)&outw.bb[0]);
-  outw.swap->GetBuffer(1, __uuidof(ID3D12Resource), (void **)&outw.bb[1]);
+    DXGI_SWAP_CHAIN_DESC swapDesc;
+    RDCEraseEl(swapDesc);
+
+    RECT rect;
+    GetClientRect(outw.wnd, &rect);
+
+    swapDesc.BufferCount = 2;
+    swapDesc.BufferDesc.Format = DXGI_FORMAT_R8G8B8A8_UNORM;
+    outw.width = swapDesc.BufferDesc.Width = rect.right - rect.left;
+    outw.height = swapDesc.BufferDesc.Height = rect.bottom - rect.top;
+    swapDesc.BufferUsage = DXGI_USAGE_RENDER_TARGET_OUTPUT;
+    swapDesc.SampleDesc.Count = 1;
+    swapDesc.SampleDesc.Quality = 0;
+    swapDesc.OutputWindow = outw.wnd;
+    swapDesc.Windowed = TRUE;
+    swapDesc.SwapEffect = DXGI_SWAP_EFFECT_FLIP_DISCARD;
+    swapDesc.Flags = 0;
+
+    HRESULT hr = S_OK;
+
+    hr = m_pFactory->CreateSwapChain(m_pDevice->GetQueue(), &swapDesc, &outw.swap);
+
+    if(FAILED(hr))
+    {
+      RDCERR("Failed to create swap chain for HWND, HRESULT: %s", ToStr(hr).c_str());
+      return 0;
+    }
+
+    outw.swap->GetBuffer(0, __uuidof(ID3D12Resource), (void **)&outw.bb[0]);
+    outw.swap->GetBuffer(1, __uuidof(ID3D12Resource), (void **)&outw.bb[1]);
+  }
+  else
+  {
+    outw.width = window.headless.width;
+    outw.height = window.headless.height;
+
+    outw.wnd = NULL;
+    outw.swap = NULL;
+  }
 
   outw.bbIdx = 0;
 
@@ -187,8 +221,7 @@ uint64_t D3D12Replay::MakeOutputWindow(WindowingData window, bool depth)
 
   outw.col = NULL;
   outw.colResolve = NULL;
-  outw.MakeRTV(depth);
-  m_pDevice->CreateRenderTargetView(outw.col, NULL, outw.rtv);
+  outw.MakeRTV(depth && window.system == WindowingSystem::Win32);
 
   outw.depth = NULL;
   if(depth)
@@ -293,6 +326,145 @@ void D3D12Replay::GetOutputWindowDimensions(uint64_t id, int32_t &w, int32_t &h)
   h = m_OutputWindows[id].height;
 }
 
+void D3D12Replay::SetOutputWindowDimensions(uint64_t id, int32_t w, int32_t h)
+{
+  if(id == 0 || m_OutputWindows.find(id) == m_OutputWindows.end())
+    return;
+
+  OutputWindow &outw = m_OutputWindows[id];
+
+  // can't resize an output with an actual window backing
+  if(outw.wnd)
+    return;
+
+  m_pDevice->ExecuteLists();
+  m_pDevice->FlushLists(true);
+
+  outw.width = w;
+  outw.height = h;
+
+  outw.MakeRTV(false);
+  outw.MakeDSV();
+
+  outw.bbIdx = 0;
+}
+
+void D3D12Replay::GetOutputWindowData(uint64_t id, bytebuf &retData)
+{
+  if(id == 0 || m_OutputWindows.find(id) == m_OutputWindows.end())
+    return;
+
+  OutputWindow &outw = m_OutputWindows[id];
+
+  if(outw.col == NULL)
+    return;
+
+  D3D12_HEAP_PROPERTIES heapProps;
+  heapProps.Type = D3D12_HEAP_TYPE_READBACK;
+  heapProps.CPUPageProperty = D3D12_CPU_PAGE_PROPERTY_UNKNOWN;
+  heapProps.MemoryPoolPreference = D3D12_MEMORY_POOL_UNKNOWN;
+  heapProps.CreationNodeMask = 1;
+  heapProps.VisibleNodeMask = 1;
+
+  D3D12_RESOURCE_DESC bufDesc;
+  bufDesc.Alignment = 0;
+  bufDesc.DepthOrArraySize = 1;
+  bufDesc.Dimension = D3D12_RESOURCE_DIMENSION_BUFFER;
+  bufDesc.Flags = D3D12_RESOURCE_FLAG_NONE;
+  bufDesc.Format = DXGI_FORMAT_UNKNOWN;
+  bufDesc.Height = 1;
+  bufDesc.Layout = D3D12_TEXTURE_LAYOUT_ROW_MAJOR;
+  bufDesc.MipLevels = 1;
+  bufDesc.SampleDesc.Count = 1;
+  bufDesc.SampleDesc.Quality = 0;
+  bufDesc.Width = 1;
+
+  D3D12_RESOURCE_DESC desc = outw.col->GetDesc();
+
+  D3D12_PLACED_SUBRESOURCE_FOOTPRINT layout = {};
+
+  m_pDevice->GetCopyableFootprints(&desc, 0, 1, 0, &layout, NULL, NULL, &bufDesc.Width);
+
+  ID3D12Resource *readback = NULL;
+  HRESULT hr = m_pDevice->CreateCommittedResource(&heapProps, D3D12_HEAP_FLAG_NONE, &bufDesc,
+                                                  D3D12_RESOURCE_STATE_COPY_DEST, NULL,
+                                                  __uuidof(ID3D12Resource), (void **)&readback);
+
+  if(SUCCEEDED(hr))
+  {
+    ID3D12GraphicsCommandList *list = m_pDevice->GetNewList();
+
+    D3D12_RESOURCE_BARRIER barrier = {};
+
+    // we know there's only one subresource, and it will be in RENDER_TARGET state
+    barrier.Type = D3D12_RESOURCE_BARRIER_TYPE_TRANSITION;
+    barrier.Flags = D3D12_RESOURCE_BARRIER_FLAG_NONE;
+    barrier.Transition.pResource = outw.col;
+    barrier.Transition.Subresource = 0;
+    barrier.Transition.StateBefore = D3D12_RESOURCE_STATE_RENDER_TARGET;
+    barrier.Transition.StateAfter = D3D12_RESOURCE_STATE_COMMON;
+
+    list->ResourceBarrier(1, &barrier);
+
+    // copy to readback buffer
+    D3D12_TEXTURE_COPY_LOCATION dst, src;
+
+    src.Type = D3D12_TEXTURE_COPY_TYPE_SUBRESOURCE_INDEX;
+    src.pResource = outw.col;
+    src.SubresourceIndex = 0;
+
+    dst.Type = D3D12_TEXTURE_COPY_TYPE_PLACED_FOOTPRINT;
+    dst.pResource = readback;
+    dst.PlacedFootprint = layout;
+
+    list->CopyTextureRegion(&dst, 0, 0, 0, &src, NULL);
+
+    // transition back
+    std::swap(barrier.Transition.StateBefore, barrier.Transition.StateAfter);
+    list->ResourceBarrier(1, &barrier);
+
+    list->Close();
+
+    m_pDevice->ExecuteLists(NULL, true);
+    m_pDevice->FlushLists();
+
+    byte *data = NULL;
+    hr = readback->Map(0, NULL, (void **)&data);
+
+    if(SUCCEEDED(hr) && data)
+    {
+      retData.resize(outw.width * outw.height * 3);
+
+      byte *dstData = retData.data();
+
+      for(int32_t row = 0; row < outw.height; row++)
+      {
+        for(int32_t x = 0; x < outw.width; x++)
+        {
+          dstData[x * 3 + 0] = data[x * 4 + 0];
+          dstData[x * 3 + 1] = data[x * 4 + 1];
+          dstData[x * 3 + 2] = data[x * 4 + 2];
+        }
+
+        data += layout.Footprint.RowPitch;
+        dstData += outw.width * 3;
+      }
+
+      readback->Unmap(0, NULL);
+    }
+    else
+    {
+      RDCERR("Couldn't map readback buffer: HRESULT: %s", ToStr(hr).c_str());
+    }
+
+    SAFE_RELEASE(readback);
+  }
+  else
+  {
+    RDCERR("Couldn't create readback buffer: HRESULT: %s", ToStr(hr).c_str());
+  }
+}
+
 void D3D12Replay::ClearOutputWindowColor(uint64_t id, FloatVector col)
 {
   if(id == 0 || m_OutputWindows.find(id) == m_OutputWindows.end())
@@ -328,7 +500,7 @@ void D3D12Replay::BindOutputWindow(uint64_t id, bool depth)
 
   m_CurrentOutputWindow = id;
 
-  if(outw.bb[0] == NULL)
+  if(outw.col == NULL)
     return;
 
   SetOutputDimensions(outw.width, outw.height);
@@ -339,6 +511,9 @@ bool D3D12Replay::IsOutputWindowVisible(uint64_t id)
   if(id == 0 || m_OutputWindows.find(id) == m_OutputWindows.end())
     return false;
 
+  if(!m_OutputWindows[id].wnd)
+    return true;
+
   return (IsWindowVisible(m_OutputWindows[id].wnd) == TRUE);
 }
 
@@ -349,7 +524,7 @@ void D3D12Replay::FlipOutputWindow(uint64_t id)
 
   OutputWindow &outw = m_OutputWindows[id];
 
-  if(m_OutputWindows[id].bb[0] == NULL)
+  if(m_OutputWindows[id].bb[0] == NULL || m_OutputWindows[id].swap == NULL)
     return;
 
   D3D12_RESOURCE_BARRIER barriers[3];
@@ -358,7 +533,7 @@ void D3D12Replay::FlipOutputWindow(uint64_t id)
   barriers[0].Transition.pResource = outw.col;
   barriers[0].Transition.StateBefore = D3D12_RESOURCE_STATE_RENDER_TARGET;
   barriers[0].Transition.StateAfter =
-      outw.depth ? D3D12_RESOURCE_STATE_RESOLVE_SOURCE : D3D12_RESOURCE_STATE_COPY_SOURCE;
+      outw.multisampled ? D3D12_RESOURCE_STATE_RESOLVE_SOURCE : D3D12_RESOURCE_STATE_COPY_SOURCE;
 
   barriers[1].Transition.pResource = outw.bb[outw.bbIdx];
   barriers[1].Transition.StateBefore = D3D12_RESOURCE_STATE_PRESENT;
@@ -371,7 +546,7 @@ void D3D12Replay::FlipOutputWindow(uint64_t id)
   ID3D12GraphicsCommandList *list = m_pDevice->GetNewList();
 
   // resolve or copy from colour to backbuffer
-  if(outw.depth)
+  if(outw.multisampled)
   {
     // transition colour to resolve source, resolve target to resolve dest, backbuffer to copy dest
     list->ResourceBarrier(3, barriers);

renderdoc/driver/d3d12/d3d12_replay.cpp

@@ -1639,7 +1639,8 @@ void D3D12Replay::RenderCheckerboard()
 
     list->IASetPrimitiveTopology(D3D_PRIMITIVE_TOPOLOGY_TRIANGLESTRIP);
 
-    list->SetPipelineState(outw.depth ? m_General.CheckerboardMSAAPipe : m_General.CheckerboardPipe);
+    list->SetPipelineState(outw.multisampled ? m_General.CheckerboardMSAAPipe
+                                             : m_General.CheckerboardPipe);
 
     list->SetGraphicsRootSignature(m_General.CheckerboardRootSig);
 

renderdoc/driver/d3d12/d3d12_replay.h

@@ -113,6 +113,8 @@ public:
   void DestroyOutputWindow(uint64_t id);
   bool CheckResizeOutputWindow(uint64_t id);
   void GetOutputWindowDimensions(uint64_t id, int32_t &w, int32_t &h);
+  void SetOutputWindowDimensions(uint64_t id, int32_t w, int32_t h);
+  void GetOutputWindowData(uint64_t id, bytebuf &retData);
   void ClearOutputWindowColor(uint64_t id, FloatVector col);
   void ClearOutputWindowDepth(uint64_t id, float depth, uint8_t stencil);
   void BindOutputWindow(uint64_t id, bool depth);
@@ -299,6 +301,7 @@ private:
     void MakeDSV();
 
     int width, height;
+    bool multisampled;
   };
 
   float m_OutputWidth = 1.0f;

renderdoc/driver/gl/cgl_platform.cpp

@@ -198,7 +198,7 @@ class CGLPlatform : public GLPlatform
 
       return ret;
     }
-    else if(window.system == WindowingSystem::Unknown)
+    else if(window.system == WindowingSystem::Unknown || window.system == WindowingSystem::Headless)
     {
       ret.nsctx = NSGL_createContext(NULL, share_context.nsctx);
 

renderdoc/driver/gl/egl_platform.cpp

@@ -125,7 +125,8 @@ class EGLPlatform : public GLPlatform
       case WindowingSystem::Xlib: win = window.xlib.window; break;
 #endif
       case WindowingSystem::Unknown:
-        // allow WindowingSystem::Unknown so that internally we can create a window-less context
+      case WindowingSystem::Headless:
+        // allow Unknown and Headless so that internally we can create a window-less context
         break;
       default: RDCERR("Unexpected window system %u", window.system); break;
     }

renderdoc/driver/gl/gl_outputwindow.cpp

@@ -100,7 +100,7 @@ bool GLReplay::CheckResizeOutputWindow(uint64_t id)
 
   OutputWindow &outw = m_OutputWindows[id];
 
-  if(outw.ctx == 0)
+  if(outw.ctx == NULL || outw.system == WindowingSystem::Headless)
     return false;
 
   int32_t w, h;
@@ -184,6 +184,9 @@ void GLReplay::FlipOutputWindow(uint64_t id)
 
   OutputWindow &outw = m_OutputWindows[id];
 
+  if(outw.system == WindowingSystem::Headless)
+    return;
+
   MakeCurrentReplayContext(&outw);
 
   WrappedOpenGL &drv = *m_pDriver;
@@ -213,7 +216,17 @@ uint64_t GLReplay::MakeOutputWindow(WindowingData window, bool depth)
   if(!win.ctx)
     return 0;
 
-  m_pDriver->m_Platform.GetOutputWindowDimensions(win, win.width, win.height);
+  win.system = window.system;
+
+  if(window.system == WindowingSystem::Headless)
+  {
+    win.width = window.headless.width;
+    win.height = window.headless.height;
+  }
+  else
+  {
+    m_pDriver->m_Platform.GetOutputWindowDimensions(win, win.width, win.height);
+  }
 
   MakeCurrentReplayContext(&win);
 
@@ -253,13 +266,106 @@ void GLReplay::GetOutputWindowDimensions(uint64_t id, int32_t &w, int32_t &h)
 
   OutputWindow &outw = m_OutputWindows[id];
 
+  if(outw.system == WindowingSystem::Headless)
+  {
+    w = outw.width;
+    h = outw.height;
+    return;
+  }
+
   m_pDriver->m_Platform.GetOutputWindowDimensions(outw, w, h);
 }
 
+void GLReplay::SetOutputWindowDimensions(uint64_t id, int32_t w, int32_t h)
+{
+  if(id == 0 || m_OutputWindows.find(id) == m_OutputWindows.end())
+    return;
+
+  OutputWindow &outw = m_OutputWindows[id];
+
+  // can't resize an output with an actual window backing
+  if(outw.system != WindowingSystem::Headless)
+    return;
+
+  outw.width = w;
+  outw.height = h;
+
+  MakeCurrentReplayContext(m_DebugCtx);
+
+  WrappedOpenGL &drv = *m_pDriver;
+
+  bool haddepth = false;
+
+  drv.glDeleteTextures(1, &outw.BlitData.backbuffer);
+  if(outw.BlitData.depthstencil)
+  {
+    haddepth = true;
+    drv.glDeleteTextures(1, &outw.BlitData.depthstencil);
+  }
+  drv.glDeleteFramebuffers(1, &outw.BlitData.windowFBO);
+
+  CreateOutputWindowBackbuffer(outw, haddepth);
+}
+
+void GLReplay::GetOutputWindowData(uint64_t id, bytebuf &retData)
+{
+  if(id == 0 || m_OutputWindows.find(id) == m_OutputWindows.end())
+    return;
+
+  OutputWindow &outw = m_OutputWindows[id];
+
+  MakeCurrentReplayContext(m_DebugCtx);
+
+  m_pDriver->glBindFramebuffer(eGL_READ_FRAMEBUFFER, outw.BlitData.windowFBO);
+  m_pDriver->glReadBuffer(eGL_BACK);
+  m_pDriver->glBindBuffer(eGL_PIXEL_PACK_BUFFER, 0);
+  m_pDriver->glPixelStorei(eGL_PACK_ROW_LENGTH, 0);
+  m_pDriver->glPixelStorei(eGL_PACK_SKIP_ROWS, 0);
+  m_pDriver->glPixelStorei(eGL_PACK_SKIP_PIXELS, 0);
+  m_pDriver->glPixelStorei(eGL_PACK_ALIGNMENT, 1);
+
+  // read as RGBA for maximum compatibility.
+  retData.resize(outw.width * outw.height * 4);
+  GL.glReadPixels(0, 0, outw.width, outw.height, eGL_RGBA, eGL_UNSIGNED_BYTE, retData.data());
+
+  // y-flip
+  for(int32_t row = 0; row < outw.height / 2; row++)
+  {
+    const uint32_t stride = outw.width * 4;
+    const int32_t fliprow = outw.height - 1 - row;
+
+    for(int32_t x = 0; x < outw.width; x++)
+    {
+      std::swap(retData[row * stride + x * 4 + 0], retData[fliprow * stride + x * 4 + 0]);
+      std::swap(retData[row * stride + x * 4 + 1], retData[fliprow * stride + x * 4 + 1]);
+      std::swap(retData[row * stride + x * 4 + 2], retData[fliprow * stride + x * 4 + 2]);
+      std::swap(retData[row * stride + x * 4 + 3], retData[fliprow * stride + x * 4 + 3]);
+    }
+  }
+
+  // compact from RGBA to RGB.
+  byte *src = retData.data();
+  byte *dst = retData.data();
+  for(int32_t row = 0; row < outw.height; row++)
+  {
+    for(int32_t x = 0; x < outw.width; x++)
+    {
+      memcpy(dst, src, 3);
+      dst += 3;
+      src += 4;
+    }
+  }
+
+  retData.resize(outw.width * outw.height * 3);
+}
+
 bool GLReplay::IsOutputWindowVisible(uint64_t id)
 {
   if(id == 0 || m_OutputWindows.find(id) == m_OutputWindows.end())
     return false;
 
+  if(m_OutputWindows[id].system == WindowingSystem::Headless)
+    return true;
+
   return m_pDriver->m_Platform.IsOutputWindowVisible(m_OutputWindows[id]);
 }

renderdoc/driver/gl/gl_replay.h

@@ -137,6 +137,8 @@ public:
   void DestroyOutputWindow(uint64_t id);
   bool CheckResizeOutputWindow(uint64_t id);
   void GetOutputWindowDimensions(uint64_t id, int32_t &w, int32_t &h);
+  void SetOutputWindowDimensions(uint64_t id, int32_t w, int32_t h);
+  void GetOutputWindowData(uint64_t id, bytebuf &retData);
   void ClearOutputWindowColor(uint64_t id, FloatVector col);
   void ClearOutputWindowDepth(uint64_t id, float depth, uint8_t stencil);
   void BindOutputWindow(uint64_t id, bool depth);
@@ -279,6 +281,7 @@ private:
       GLuint readFBO = 0;
     } BlitData;
 
+    WindowingSystem system = WindowingSystem::Headless;
     int width = 1, height = 1;
   };
 

renderdoc/driver/gl/glx_platform.cpp

@@ -151,7 +151,7 @@ class GLXPlatform : public GLPlatform
           "support compiled in");
 #endif
     }
-    else if(window.system == WindowingSystem::Unknown)
+    else if(window.system == WindowingSystem::Unknown || window.system == WindowingSystem::Headless)
     {
       // allow WindowingSystem::Unknown so that internally we can create a window-less context
       dpy = RenderDoc::Inst().GetGlobalEnvironment().xlibDisplay;

renderdoc/driver/gl/wgl_platform.cpp

@@ -107,12 +107,15 @@ class WGLPlatform : public GLPlatform
     if(!WGL.wglGetPixelFormatAttribivARB || !WGL.wglCreateContextAttribsARB)
       return ret;
 
-    RDCASSERT(window.system == WindowingSystem::Win32 || window.system == WindowingSystem::Unknown,
+    RDCASSERT(window.system == WindowingSystem::Win32 || window.system == WindowingSystem::Unknown ||
+                  window.system == WindowingSystem::Headless,
               window.system);
 
-    HWND w = window.win32.window;
+    HWND w = NULL;
 
-    if(w == NULL)
+    if(window.system == WindowingSystem::Win32)
+      w = window.win32.window;
+    else
       w = CreateWindowEx(WS_EX_CLIENTEDGE, WINDOW_CLASS_NAME, L"", WS_OVERLAPPEDWINDOW,
                          CW_USEDEFAULT, CW_USEDEFAULT, CW_USEDEFAULT, CW_USEDEFAULT, NULL, NULL,
                          GetModuleHandle(NULL), NULL);

renderdoc/driver/vulkan/vk_android.cpp

@@ -77,6 +77,13 @@ void VulkanReplay::GetOutputWindowDimensions(uint64_t id, int32_t &w, int32_t &h
 
   OutputWindow &outw = m_OutputWindows[id];
 
+  if(outw.m_WindowSystem == WindowingSystem::Headless)
+  {
+    w = outw.width;
+    h = outw.height;
+    return;
+  }
+
   w = ANativeWindow_getWidth(outw.wnd);
   h = ANativeWindow_getHeight(outw.wnd);
 }

renderdoc/driver/vulkan/vk_apple.cpp

@@ -83,6 +83,13 @@ void VulkanReplay::GetOutputWindowDimensions(uint64_t id, int32_t &w, int32_t &h
 
   OutputWindow &outw = m_OutputWindows[id];
 
+  if(outw.m_WindowSystem == WindowingSystem::Headless)
+  {
+    w = outw.width;
+    h = outw.height;
+    return;
+  }
+
   w = getMetalLayerWidth(outw.wnd);
   h = getMetalLayerHeight(outw.wnd);
 }

renderdoc/driver/vulkan/vk_ggp.cpp

@@ -76,6 +76,13 @@ void VulkanReplay::GetOutputWindowDimensions(uint64_t id, int32_t &w, int32_t &h
 
   OutputWindow &outw = m_OutputWindows[id];
 
+  if(outw.m_WindowSystem == WindowingSystem::Headless)
+  {
+    w = outw.width;
+    h = outw.height;
+    return;
+  }
+
   RDCLOG("Window system is GGP (%d), size is %d, %d", outw.m_WindowSystem, outw.width, outw.height);
   // No window, specify default resolution.
   w = outw.width != 0 ? outw.width : 1920;

renderdoc/driver/vulkan/vk_linux.cpp

@@ -203,6 +203,13 @@ void VulkanReplay::GetOutputWindowDimensions(uint64_t id, int32_t &w, int32_t &h
 
   OutputWindow &outw = m_OutputWindows[id];
 
+  if(outw.m_WindowSystem == WindowingSystem::Headless)
+  {
+    w = outw.width;
+    h = outw.height;
+    return;
+  }
+
 #if ENABLED(RDOC_XLIB)
   if(outw.m_WindowSystem == WindowingSystem::Xlib)
   {

renderdoc/driver/vulkan/vk_outputwindow.cpp

@@ -85,14 +85,6 @@ VulkanReplay::OutputWindow::OutputWindow()
       VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT;
 }
 
-void VulkanReplay::OutputWindow::SetCol(VkDeviceMemory mem, VkImage img)
-{
-}
-
-void VulkanReplay::OutputWindow::SetDS(VkDeviceMemory mem, VkImage img)
-{
-}
-
 void VulkanReplay::OutputWindow::Destroy(WrappedVulkan *driver, VkDevice device)
 {
   const VkLayerDispatchTable *vt = ObjDisp(device);
@@ -197,7 +189,7 @@ void VulkanReplay::OutputWindow::Create(WrappedVulkan *driver, VkDevice device,
 
   fresh = true;
 
-  if(surface == VK_NULL_HANDLE)
+  if(surface == VK_NULL_HANDLE && m_WindowSystem != WindowingSystem::Headless)
   {
     CreateSurface(inst);
 
@@ -211,182 +203,185 @@ void VulkanReplay::OutputWindow::Create(WrappedVulkan *driver, VkDevice device,
 
   VkResult vkr = VK_SUCCESS;
 
-  VkSurfaceCapabilitiesKHR capabilities;
-
-  ObjDisp(inst)->GetPhysicalDeviceSurfaceCapabilitiesKHR(Unwrap(phys), Unwrap(surface),
-                                                         &capabilities);
-
-  RDCASSERT(capabilities.supportedUsageFlags & VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT);
-  // AMD didn't report this capability for a while. If the assert fires for you, update
-  // your drivers!
-  RDCASSERT(capabilities.supportedUsageFlags & VK_IMAGE_USAGE_TRANSFER_DST_BIT);
-
-  RDCASSERT(capabilities.minImageCount <= 2 &&
-            (2 <= capabilities.maxImageCount || capabilities.maxImageCount == 0));
-
-  // check format and present mode from driver
+  if(m_WindowSystem != WindowingSystem::Headless)
   {
-    uint32_t numFormats = 0;
+    VkSurfaceCapabilitiesKHR capabilities;
 
-    vkr = ObjDisp(inst)->GetPhysicalDeviceSurfaceFormatsKHR(Unwrap(phys), Unwrap(surface),
-                                                            &numFormats, NULL);
-    RDCASSERTEQUAL(vkr, VK_SUCCESS);
+    ObjDisp(inst)->GetPhysicalDeviceSurfaceCapabilitiesKHR(Unwrap(phys), Unwrap(surface),
+                                                           &capabilities);
 
-    if(numFormats > 0)
+    RDCASSERT(capabilities.supportedUsageFlags & VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT);
+    // AMD didn't report this capability for a while. If the assert fires for you, update
+    // your drivers!
+    RDCASSERT(capabilities.supportedUsageFlags & VK_IMAGE_USAGE_TRANSFER_DST_BIT);
+
+    RDCASSERT(capabilities.minImageCount <= 2 &&
+              (2 <= capabilities.maxImageCount || capabilities.maxImageCount == 0));
+
+    // check format and present mode from driver
     {
-      VkSurfaceFormatKHR *formats = new VkSurfaceFormatKHR[numFormats];
+      uint32_t numFormats = 0;
 
       vkr = ObjDisp(inst)->GetPhysicalDeviceSurfaceFormatsKHR(Unwrap(phys), Unwrap(surface),
-                                                              &numFormats, formats);
+                                                              &numFormats, NULL);
       RDCASSERTEQUAL(vkr, VK_SUCCESS);
 
-      if(numFormats == 1 && formats[0].format == VK_FORMAT_UNDEFINED)
+      if(numFormats > 0)
       {
-        // 1 entry with undefined means no preference, just use our default
-        imformat = VK_FORMAT_B8G8R8A8_SRGB;
-        imcolspace = VK_COLORSPACE_SRGB_NONLINEAR_KHR;
-      }
-      else
-      {
-        // try and find a format with SRGB correction
-        imformat = VK_FORMAT_UNDEFINED;
-        imcolspace = formats[0].colorSpace;
+        VkSurfaceFormatKHR *formats = new VkSurfaceFormatKHR[numFormats];
 
-        for(uint32_t i = 0; i < numFormats; i++)
+        vkr = ObjDisp(inst)->GetPhysicalDeviceSurfaceFormatsKHR(Unwrap(phys), Unwrap(surface),
+                                                                &numFormats, formats);
+        RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+        if(numFormats == 1 && formats[0].format == VK_FORMAT_UNDEFINED)
         {
-          if(IsSRGBFormat(formats[i].format))
+          // 1 entry with undefined means no preference, just use our default
+          imformat = VK_FORMAT_B8G8R8A8_SRGB;
+          imcolspace = VK_COLORSPACE_SRGB_NONLINEAR_KHR;
+        }
+        else
+        {
+          // try and find a format with SRGB correction
+          imformat = VK_FORMAT_UNDEFINED;
+          imcolspace = formats[0].colorSpace;
+
+          for(uint32_t i = 0; i < numFormats; i++)
           {
-            imformat = formats[i].format;
-            imcolspace = formats[i].colorSpace;
-            RDCASSERT(imcolspace == VK_COLORSPACE_SRGB_NONLINEAR_KHR);
-            break;
+            if(IsSRGBFormat(formats[i].format))
+            {
+              imformat = formats[i].format;
+              imcolspace = formats[i].colorSpace;
+              RDCASSERT(imcolspace == VK_COLORSPACE_SRGB_NONLINEAR_KHR);
+              break;
+            }
+          }
+
+          if(imformat == VK_FORMAT_UNDEFINED)
+          {
+            RDCWARN("Couldn't find SRGB correcting output swapchain format");
+            imformat = formats[0].format;
           }
         }
 
-        if(imformat == VK_FORMAT_UNDEFINED)
-        {
-          RDCWARN("Couldn't find SRGB correcting output swapchain format");
-          imformat = formats[0].format;
-        }
+        SAFE_DELETE_ARRAY(formats);
       }
 
-      SAFE_DELETE_ARRAY(formats);
-    }
-
-    uint32_t numModes = 0;
-
-    vkr = ObjDisp(inst)->GetPhysicalDeviceSurfacePresentModesKHR(Unwrap(phys), Unwrap(surface),
-                                                                 &numModes, NULL);
-    RDCASSERTEQUAL(vkr, VK_SUCCESS);
-
-    if(numModes > 0)
-    {
-      VkPresentModeKHR *modes = new VkPresentModeKHR[numModes];
+      uint32_t numModes = 0;
 
       vkr = ObjDisp(inst)->GetPhysicalDeviceSurfacePresentModesKHR(Unwrap(phys), Unwrap(surface),
-                                                                   &numModes, modes);
+                                                                   &numModes, NULL);
       RDCASSERTEQUAL(vkr, VK_SUCCESS);
 
-      // If mailbox mode is available, use it, as is the lowest-latency non-
-      // tearing mode.  If not, try IMMEDIATE which will usually be available,
-      // and is fastest (though it tears).  If not, fall back to FIFO which is
-      // always available.
-      for(size_t i = 0; i < numModes; i++)
+      if(numModes > 0)
       {
-        if(modes[i] == VK_PRESENT_MODE_MAILBOX_KHR)
+        VkPresentModeKHR *modes = new VkPresentModeKHR[numModes];
+
+        vkr = ObjDisp(inst)->GetPhysicalDeviceSurfacePresentModesKHR(Unwrap(phys), Unwrap(surface),
+                                                                     &numModes, modes);
+        RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+        // If mailbox mode is available, use it, as is the lowest-latency non-
+        // tearing mode.  If not, try IMMEDIATE which will usually be available,
+        // and is fastest (though it tears).  If not, fall back to FIFO which is
+        // always available.
+        for(size_t i = 0; i < numModes; i++)
         {
-          presentmode = VK_PRESENT_MODE_MAILBOX_KHR;
-          break;
+          if(modes[i] == VK_PRESENT_MODE_MAILBOX_KHR)
+          {
+            presentmode = VK_PRESENT_MODE_MAILBOX_KHR;
+            break;
+          }
+
+          if(modes[i] == VK_PRESENT_MODE_IMMEDIATE_KHR)
+            presentmode = VK_PRESENT_MODE_IMMEDIATE_KHR;
         }
 
-        if(modes[i] == VK_PRESENT_MODE_IMMEDIATE_KHR)
-          presentmode = VK_PRESENT_MODE_IMMEDIATE_KHR;
+        SAFE_DELETE_ARRAY(modes);
       }
-
-      SAFE_DELETE_ARRAY(modes);
     }
+
+    VkBool32 supported = false;
+    ObjDisp(inst)->GetPhysicalDeviceSurfaceSupportKHR(Unwrap(phys), driver->GetQFamilyIdx(),
+                                                      Unwrap(surface), &supported);
+
+    // can't really recover from this anyway
+    RDCASSERT(supported);
+
+    VkSwapchainCreateInfoKHR swapInfo = {
+        VK_STRUCTURE_TYPE_SWAPCHAIN_CREATE_INFO_KHR,
+        NULL,
+        0,
+        Unwrap(surface),
+        2,
+        imformat,
+        imcolspace,
+        {width, height},
+        1,
+        VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT,
+        VK_SHARING_MODE_EXCLUSIVE,
+        0,
+        NULL,
+        VK_SURFACE_TRANSFORM_IDENTITY_BIT_KHR,
+        VK_COMPOSITE_ALPHA_OPAQUE_BIT_KHR,
+        presentmode,
+        true,
+        Unwrap(old),
+    };
+
+    vkr = vt->CreateSwapchainKHR(Unwrap(device), &swapInfo, NULL, &swap);
+    RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+    if(old != VK_NULL_HANDLE)
+    {
+      vt->DestroySwapchainKHR(Unwrap(device), Unwrap(old), NULL);
+      GetResourceManager()->ReleaseWrappedResource(old);
+    }
+
+    if(swap == VK_NULL_HANDLE)
+    {
+      RDCERR("Failed to create swapchain. %d consecutive failures!", failures);
+      failures++;
+
+      // do some sort of backoff.
+
+      // the first time, try to recreate again next frame
+      if(failures == 1)
+        recreatePause = 0;
+      // the next few times, wait 200 'frames' between attempts
+      else if(failures < 10)
+        recreatePause = 100;
+      // otherwise, only reattempt very infrequently. A resize will
+      // always retrigger a recreate, so ew probably don't want to
+      // try again
+      else
+        recreatePause = 1000;
+
+      return;
+    }
+
+    failures = 0;
+
+    GetResourceManager()->WrapResource(Unwrap(device), swap);
+
+    vkr = vt->GetSwapchainImagesKHR(Unwrap(device), Unwrap(swap), &numImgs, NULL);
+    RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+    VkImage *imgs = new VkImage[numImgs];
+    vkr = vt->GetSwapchainImagesKHR(Unwrap(device), Unwrap(swap), &numImgs, imgs);
+    RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+    for(size_t i = 0; i < numImgs; i++)
+    {
+      colimg[i] = imgs[i];
+      GetResourceManager()->WrapResource(Unwrap(device), colimg[i]);
+      colBarrier[i].image = Unwrap(colimg[i]);
+      colBarrier[i].oldLayout = colBarrier[i].newLayout = VK_IMAGE_LAYOUT_UNDEFINED;
+    }
+
+    delete[] imgs;
   }
 
-  VkBool32 supported = false;
-  ObjDisp(inst)->GetPhysicalDeviceSurfaceSupportKHR(Unwrap(phys), driver->GetQFamilyIdx(),
-                                                    Unwrap(surface), &supported);
-
-  // can't really recover from this anyway
-  RDCASSERT(supported);
-
-  VkSwapchainCreateInfoKHR swapInfo = {
-      VK_STRUCTURE_TYPE_SWAPCHAIN_CREATE_INFO_KHR,
-      NULL,
-      0,
-      Unwrap(surface),
-      2,
-      imformat,
-      imcolspace,
-      {width, height},
-      1,
-      VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT,
-      VK_SHARING_MODE_EXCLUSIVE,
-      0,
-      NULL,
-      VK_SURFACE_TRANSFORM_IDENTITY_BIT_KHR,
-      VK_COMPOSITE_ALPHA_OPAQUE_BIT_KHR,
-      presentmode,
-      true,
-      Unwrap(old),
-  };
-
-  vkr = vt->CreateSwapchainKHR(Unwrap(device), &swapInfo, NULL, &swap);
-  RDCASSERTEQUAL(vkr, VK_SUCCESS);
-
-  if(old != VK_NULL_HANDLE)
-  {
-    vt->DestroySwapchainKHR(Unwrap(device), Unwrap(old), NULL);
-    GetResourceManager()->ReleaseWrappedResource(old);
-  }
-
-  if(swap == VK_NULL_HANDLE)
-  {
-    RDCERR("Failed to create swapchain. %d consecutive failures!", failures);
-    failures++;
-
-    // do some sort of backoff.
-
-    // the first time, try to recreate again next frame
-    if(failures == 1)
-      recreatePause = 0;
-    // the next few times, wait 200 'frames' between attempts
-    else if(failures < 10)
-      recreatePause = 100;
-    // otherwise, only reattempt very infrequently. A resize will
-    // always retrigger a recreate, so ew probably don't want to
-    // try again
-    else
-      recreatePause = 1000;
-
-    return;
-  }
-
-  failures = 0;
-
-  GetResourceManager()->WrapResource(Unwrap(device), swap);
-
-  vkr = vt->GetSwapchainImagesKHR(Unwrap(device), Unwrap(swap), &numImgs, NULL);
-  RDCASSERTEQUAL(vkr, VK_SUCCESS);
-
-  VkImage *imgs = new VkImage[numImgs];
-  vkr = vt->GetSwapchainImagesKHR(Unwrap(device), Unwrap(swap), &numImgs, imgs);
-  RDCASSERTEQUAL(vkr, VK_SUCCESS);
-
-  for(size_t i = 0; i < numImgs; i++)
-  {
-    colimg[i] = imgs[i];
-    GetResourceManager()->WrapResource(Unwrap(device), colimg[i]);
-    colBarrier[i].image = Unwrap(colimg[i]);
-    colBarrier[i].oldLayout = colBarrier[i].newLayout = VK_IMAGE_LAYOUT_UNDEFINED;
-  }
-
-  delete[] imgs;
-
   curidx = 0;
 
   // for our 'fake' backbuffer, create in RGBA8
@@ -639,6 +634,143 @@ void VulkanReplay::OutputWindow::Create(WrappedVulkan *driver, VkDevice device,
   }
 }
 
+void VulkanReplay::GetOutputWindowData(uint64_t id, bytebuf &retData)
+{
+  if(id == 0 || m_OutputWindows.find(id) == m_OutputWindows.end())
+    return;
+
+  OutputWindow &outw = m_OutputWindows[id];
+
+  VkDevice device = m_pDriver->GetDev();
+  VkCommandBuffer cmd = m_pDriver->GetNextCmd();
+
+  const VkLayerDispatchTable *vt = ObjDisp(device);
+
+  vt->DeviceWaitIdle(Unwrap(device));
+
+  VkBuffer readbackBuf = VK_NULL_HANDLE;
+
+  VkResult vkr = VK_SUCCESS;
+
+  // create readback buffer
+  VkBufferCreateInfo bufInfo = {
+      VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO,
+      NULL,
+      0,
+      GetByteSize(outw.width, outw.height, 1, VK_FORMAT_R8G8B8A8_UNORM, 0),
+      VK_BUFFER_USAGE_TRANSFER_DST_BIT,
+  };
+  vt->CreateBuffer(Unwrap(device), &bufInfo, NULL, &readbackBuf);
+  RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+  VkMemoryRequirements mrq = {0};
+
+  vt->GetBufferMemoryRequirements(Unwrap(device), readbackBuf, &mrq);
+
+  VkMemoryAllocateInfo allocInfo = {
+      VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, NULL, bufInfo.size,
+      m_pDriver->GetReadbackMemoryIndex(mrq.memoryTypeBits),
+  };
+
+  VkDeviceMemory readbackMem = VK_NULL_HANDLE;
+  vkr = vt->AllocateMemory(Unwrap(device), &allocInfo, NULL, &readbackMem);
+  RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+  vkr = vt->BindBufferMemory(Unwrap(device), readbackBuf, readbackMem, 0);
+  RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+  VkCommandBufferBeginInfo beginInfo = {VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, NULL,
+                                        VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT};
+
+  // do image copy
+  vkr = vt->BeginCommandBuffer(Unwrap(cmd), &beginInfo);
+  RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+  VkBufferImageCopy cpy = {
+      0,
+      0,
+      0,
+      {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1},
+      {
+          0, 0, 0,
+      },
+      {outw.width, outw.height, 1},
+  };
+
+  outw.bbBarrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
+  outw.bbBarrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
+
+  DoPipelineBarrier(cmd, 1, &outw.bbBarrier);
+
+  vt->CmdCopyImageToBuffer(Unwrap(cmd), Unwrap(outw.bb), VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
+                           readbackBuf, 1, &cpy);
+
+  outw.bbBarrier.oldLayout = outw.bbBarrier.newLayout;
+  outw.bbBarrier.srcAccessMask = outw.bbBarrier.dstAccessMask;
+
+  vkr = vt->EndCommandBuffer(Unwrap(cmd));
+  RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+  m_pDriver->SubmitCmds();
+  m_pDriver->FlushQ();    // need to wait so we can readback
+
+  // map memory and readback
+  byte *pData = NULL;
+  vkr = vt->MapMemory(Unwrap(device), readbackMem, 0, bufInfo.size, 0, (void **)&pData);
+  RDCASSERTEQUAL(vkr, VK_SUCCESS);
+  RDCASSERT(pData != NULL);
+
+  VkMappedMemoryRange range = {
+      VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE, NULL, readbackMem, 0, bufInfo.size,
+  };
+
+  vkr = vt->InvalidateMappedMemoryRanges(Unwrap(device), 1, &range);
+  RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+  {
+    retData.resize(outw.width * outw.height * 3);
+
+    byte *src = (byte *)pData;
+    byte *dst = retData.data();
+
+    for(uint32_t row = 0; row < outw.height; row++)
+    {
+      for(uint32_t x = 0; x < outw.width; x++)
+      {
+        dst[x * 3 + 0] = src[x * 4 + 0];
+        dst[x * 3 + 1] = src[x * 4 + 1];
+        dst[x * 3 + 2] = src[x * 4 + 2];
+      }
+
+      src += outw.width * 4;
+      dst += outw.width * 3;
+    }
+  }
+
+  vt->UnmapMemory(Unwrap(device), readbackMem);
+
+  // delete all
+  vt->DestroyBuffer(Unwrap(device), readbackBuf, NULL);
+  vt->FreeMemory(Unwrap(device), readbackMem, NULL);
+}
+
+void VulkanReplay::SetOutputWindowDimensions(uint64_t id, int32_t w, int32_t h)
+{
+  if(id == 0 || m_OutputWindows.find(id) == m_OutputWindows.end())
+    return;
+
+  OutputWindow &outw = m_OutputWindows[id];
+
+  // can't resize an output with an actual window backing
+  if(outw.m_WindowSystem != WindowingSystem::Headless)
+    return;
+
+  outw.width = w;
+  outw.height = h;
+
+  outw.Create(m_pDriver, m_pDriver->GetDev(), outw.hasDepth);
+}
+
 bool VulkanReplay::CheckResizeOutputWindow(uint64_t id)
 {
   if(id == 0 || m_OutputWindows.find(id) == m_OutputWindows.end())
@@ -646,7 +778,8 @@ bool VulkanReplay::CheckResizeOutputWindow(uint64_t id)
 
   OutputWindow &outw = m_OutputWindows[id];
 
-  if(outw.m_WindowSystem == WindowingSystem::Unknown)
+  if(outw.m_WindowSystem == WindowingSystem::Unknown ||
+     outw.m_WindowSystem == WindowingSystem::Headless)
     return false;
 
   int32_t w, h;
@@ -687,7 +820,7 @@ void VulkanReplay::BindOutputWindow(uint64_t id, bool depth)
 
   // if the swapchain failed to create, do nothing. We will try to recreate it
   // again in CheckResizeOutputWindow (once per render 'frame')
-  if(outw.swap == VK_NULL_HANDLE)
+  if(outw.m_WindowSystem != WindowingSystem::Headless && outw.swap == VK_NULL_HANDLE)
     return;
 
   m_DebugWidth = (int32_t)outw.width;
@@ -696,53 +829,58 @@ void VulkanReplay::BindOutputWindow(uint64_t id, bool depth)
   VkDevice dev = m_pDriver->GetDev();
   VkCommandBuffer cmd = m_pDriver->GetNextCmd();
   const VkLayerDispatchTable *vt = ObjDisp(dev);
+  VkResult vkr = VK_SUCCESS;
 
-  // semaphore is short lived, so not wrapped, if it's cached (ideally)
-  // then it should be wrapped
-  VkSemaphore sem;
-  VkPipelineStageFlags stage = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT;
-  VkSemaphoreCreateInfo semInfo = {VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO, NULL, 0};
-
-  VkResult vkr = vt->CreateSemaphore(Unwrap(dev), &semInfo, NULL, &sem);
-  RDCASSERTEQUAL(vkr, VK_SUCCESS);
-
-  vkr = vt->AcquireNextImageKHR(Unwrap(dev), Unwrap(outw.swap), UINT64_MAX, sem, VK_NULL_HANDLE,
-                                &outw.curidx);
-
-  if(vkr == VK_ERROR_OUT_OF_DATE_KHR)
+  // if we have a swapchain, acquire the next image.
+  if(outw.swap != VK_NULL_HANDLE)
   {
-    // force a swapchain recreate.
-    outw.width = 0;
-    outw.height = 0;
+    // semaphore is short lived, so not wrapped, if it's cached (ideally)
+    // then it should be wrapped
+    VkSemaphore sem;
+    VkPipelineStageFlags stage = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT;
+    VkSemaphoreCreateInfo semInfo = {VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO, NULL, 0};
 
-    CheckResizeOutputWindow(id);
+    vkr = vt->CreateSemaphore(Unwrap(dev), &semInfo, NULL, &sem);
+    RDCASSERTEQUAL(vkr, VK_SUCCESS);
 
-    // then try again to acquire.
     vkr = vt->AcquireNextImageKHR(Unwrap(dev), Unwrap(outw.swap), UINT64_MAX, sem, VK_NULL_HANDLE,
                                   &outw.curidx);
+
+    if(vkr == VK_ERROR_OUT_OF_DATE_KHR)
+    {
+      // force a swapchain recreate.
+      outw.width = 0;
+      outw.height = 0;
+
+      CheckResizeOutputWindow(id);
+
+      // then try again to acquire.
+      vkr = vt->AcquireNextImageKHR(Unwrap(dev), Unwrap(outw.swap), UINT64_MAX, sem, VK_NULL_HANDLE,
+                                    &outw.curidx);
+    }
+
+    RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+    VkSubmitInfo submitInfo = {
+        VK_STRUCTURE_TYPE_SUBMIT_INFO,
+        NULL,
+        1,
+        &sem,
+        &stage,
+        0,
+        NULL,    // cmd buffers
+        0,
+        NULL,    // signal semaphores
+    };
+
+    vkr = vt->QueueSubmit(Unwrap(m_pDriver->GetQ()), 1, &submitInfo, VK_NULL_HANDLE);
+    RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+    vt->QueueWaitIdle(Unwrap(m_pDriver->GetQ()));
+
+    vt->DestroySemaphore(Unwrap(dev), sem, NULL);
   }
 
-  RDCASSERTEQUAL(vkr, VK_SUCCESS);
-
-  VkSubmitInfo submitInfo = {
-      VK_STRUCTURE_TYPE_SUBMIT_INFO,
-      NULL,
-      1,
-      &sem,
-      &stage,
-      0,
-      NULL,    // cmd buffers
-      0,
-      NULL,    // signal semaphores
-  };
-
-  vkr = vt->QueueSubmit(Unwrap(m_pDriver->GetQ()), 1, &submitInfo, VK_NULL_HANDLE);
-  RDCASSERTEQUAL(vkr, VK_SUCCESS);
-
-  vt->QueueWaitIdle(Unwrap(m_pDriver->GetQ()));
-
-  vt->DestroySemaphore(Unwrap(dev), sem, NULL);
-
   VkCommandBufferBeginInfo beginInfo = {VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, NULL,
                                         VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT};
 
@@ -775,7 +913,8 @@ void VulkanReplay::BindOutputWindow(uint64_t id, bool depth)
   outw.colBarrier[outw.curidx].dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
 
   DoPipelineBarrier(cmd, 1, &outw.bbBarrier);
-  DoPipelineBarrier(cmd, 1, &outw.colBarrier[outw.curidx]);
+  if(outw.colimg[0] != VK_NULL_HANDLE)
+    DoPipelineBarrier(cmd, 1, &outw.colBarrier[outw.curidx]);
   if(outw.dsimg != VK_NULL_HANDLE)
     DoPipelineBarrier(cmd, 1, &outw.depthBarrier);
 
@@ -802,7 +941,7 @@ void VulkanReplay::ClearOutputWindowColor(uint64_t id, FloatVector col)
 
   // if the swapchain failed to create, do nothing. We will try to recreate it
   // again in CheckResizeOutputWindow (once per render 'frame')
-  if(outw.swap == VK_NULL_HANDLE)
+  if(outw.m_WindowSystem != WindowingSystem::Headless && outw.swap == VK_NULL_HANDLE)
     return;
 
   VkDevice dev = m_pDriver->GetDev();
@@ -852,7 +991,7 @@ void VulkanReplay::ClearOutputWindowDepth(uint64_t id, float depth, uint8_t sten
 
   // if the swapchain failed to create, do nothing. We will try to recreate it
   // again in CheckResizeOutputWindow (once per render 'frame')
-  if(outw.swap == VK_NULL_HANDLE)
+  if(outw.m_WindowSystem != WindowingSystem::Headless && outw.swap == VK_NULL_HANDLE)
     return;
 
   VkDevice dev = m_pDriver->GetDev();
@@ -1059,13 +1198,24 @@ uint64_t VulkanReplay::MakeOutputWindow(WindowingData window, bool depth)
   m_OutputWinID++;
 
   m_OutputWindows[id].m_WindowSystem = window.system;
-  m_OutputWindows[id].SetWindowHandle(window);
   m_OutputWindows[id].m_ResourceManager = GetResourceManager();
 
+  if(window.system != WindowingSystem::Unknown && window.system != WindowingSystem::Headless)
+    m_OutputWindows[id].SetWindowHandle(window);
+
   if(window.system != WindowingSystem::Unknown)
   {
     int32_t w, h;
-    GetOutputWindowDimensions(id, w, h);
+
+    if(window.system == WindowingSystem::Headless)
+    {
+      w = window.headless.width;
+      h = window.headless.height;
+    }
+    else
+    {
+      GetOutputWindowDimensions(id, w, h);
+    }
 
     m_OutputWindows[id].width = w;
     m_OutputWindows[id].height = h;

renderdoc/driver/vulkan/vk_posix.cpp

@@ -36,6 +36,9 @@ bool VulkanReplay::IsOutputWindowVisible(uint64_t id)
   if(id == 0 || m_OutputWindows.find(id) == m_OutputWindows.end())
     return false;
 
+  if(m_OutputWindows[id].m_WindowSystem == WindowingSystem::Headless)
+    return true;
+
   VULKANNOTIMP("Optimisation missing - output window always returning true");
 
   return true;

renderdoc/driver/vulkan/vk_rendermesh.cpp

@@ -370,7 +370,7 @@ void VulkanReplay::RenderMesh(uint32_t eventId, const vector<MeshFormat> &second
 
   // if the swapchain failed to create, do nothing. We will try to recreate it
   // again in CheckResizeOutputWindow (once per render 'frame')
-  if(outw.swap == VK_NULL_HANDLE)
+  if(outw.m_WindowSystem != WindowingSystem::Headless && outw.swap == VK_NULL_HANDLE)
     return;
 
   VkDevice dev = m_pDriver->GetDev();

renderdoc/driver/vulkan/vk_rendertexture.cpp

@@ -129,7 +129,7 @@ bool VulkanReplay::RenderTexture(TextureDisplay cfg)
 
   // if the swapchain failed to create, do nothing. We will try to recreate it
   // again in CheckResizeOutputWindow (once per render 'frame')
-  if(outw.swap == VK_NULL_HANDLE)
+  if(outw.m_WindowSystem != WindowingSystem::Headless && outw.swap == VK_NULL_HANDLE)
     return false;
 
   VkRenderPassBeginInfo rpbegin = {

renderdoc/driver/vulkan/vk_replay.cpp

@@ -580,7 +580,7 @@ void VulkanReplay::RenderCheckerboard()
 
   // if the swapchain failed to create, do nothing. We will try to recreate it
   // again in CheckResizeOutputWindow (once per render 'frame')
-  if(outw.swap == VK_NULL_HANDLE)
+  if(outw.m_WindowSystem != WindowingSystem::Headless && outw.swap == VK_NULL_HANDLE)
     return;
 
   VkDevice dev = m_pDriver->GetDev();
@@ -706,7 +706,7 @@ void VulkanReplay::RenderHighlightBox(float w, float h, float scale)
 
   // if the swapchain failed to create, do nothing. We will try to recreate it
   // again in CheckResizeOutputWindow (once per render 'frame')
-  if(outw.swap == VK_NULL_HANDLE)
+  if(outw.m_WindowSystem != WindowingSystem::Headless && outw.swap == VK_NULL_HANDLE)
     return;
 
   VkDevice dev = m_pDriver->GetDev();

renderdoc/driver/vulkan/vk_replay.h

@@ -274,6 +274,8 @@ public:
   void DestroyOutputWindow(uint64_t id);
   bool CheckResizeOutputWindow(uint64_t id);
   void GetOutputWindowDimensions(uint64_t id, int32_t &w, int32_t &h);
+  void SetOutputWindowDimensions(uint64_t id, int32_t w, int32_t h);
+  void GetOutputWindowData(uint64_t id, bytebuf &retData);
   void ClearOutputWindowColor(uint64_t id, FloatVector col);
   void ClearOutputWindowDepth(uint64_t id, float depth, uint8_t stencil);
   void BindOutputWindow(uint64_t id, bool depth);
@@ -403,8 +405,6 @@ private:
   {
     OutputWindow();
 
-    void SetCol(VkDeviceMemory mem, VkImage img);
-    void SetDS(VkDeviceMemory mem, VkImage img);
     void Create(WrappedVulkan *driver, VkDevice device, bool depth);
     void Destroy(WrappedVulkan *driver, VkDevice device);
 

renderdoc/driver/vulkan/vk_win32.cpp

@@ -60,6 +60,13 @@ void VulkanReplay::GetOutputWindowDimensions(uint64_t id, int32_t &w, int32_t &h
 
   OutputWindow &outw = m_OutputWindows[id];
 
+  if(outw.m_WindowSystem == WindowingSystem::Headless)
+  {
+    w = outw.width;
+    h = outw.height;
+    return;
+  }
+
   RECT rect = {0};
   GetClientRect(outw.wnd, &rect);
   w = rect.right - rect.left;
@@ -71,6 +78,9 @@ bool VulkanReplay::IsOutputWindowVisible(uint64_t id)
   if(id == 0 || m_OutputWindows.find(id) == m_OutputWindows.end())
     return false;
 
+  if(m_OutputWindows[id].m_WindowSystem == WindowingSystem::Headless)
+    return true;
+
   return (IsWindowVisible(m_OutputWindows[id].wnd) == TRUE);
 }
 

renderdoc/replay/replay_controller.h

@@ -43,6 +43,8 @@ public:
   void SetTextureDisplay(const TextureDisplay &o);
   void SetMeshDisplay(const MeshDisplay &o);
   void SetDimensions(int32_t width, int32_t height);
+  bytebuf ReadbackOutputTexture();
+  rdcpair<int32_t, int32_t> GetDimensions();
 
   void ClearThumbnails();
   bool AddThumbnail(WindowingData window, ResourceId texID, CompType typeHint);

renderdoc/replay/replay_driver.h

@@ -186,7 +186,9 @@ public:
   virtual uint64_t MakeOutputWindow(WindowingData window, bool depth) = 0;
   virtual void DestroyOutputWindow(uint64_t id) = 0;
   virtual bool CheckResizeOutputWindow(uint64_t id) = 0;
+  virtual void SetOutputWindowDimensions(uint64_t id, int32_t w, int32_t h) = 0;
   virtual void GetOutputWindowDimensions(uint64_t id, int32_t &w, int32_t &h) = 0;
+  virtual void GetOutputWindowData(uint64_t id, bytebuf &retData) = 0;
   virtual void ClearOutputWindowColor(uint64_t id, FloatVector col) = 0;
   virtual void ClearOutputWindowDepth(uint64_t id, float depth, uint8_t stencil) = 0;
   virtual void BindOutputWindow(uint64_t id, bool depth) = 0;

renderdoc/replay/replay_output.cpp

@@ -116,17 +116,6 @@ ReplayOutput::ReplayOutput(ReplayController *parent, WindowingData window, Repla
     RenderDoc::Inst().GetCrashHandler()->RegisterMemoryRegion(this, sizeof(ReplayController));
 }
 
-void ReplayOutput::SetDimensions(int32_t width, int32_t height)
-{
-  CHECK_REPLAY_THREAD();
-
-  if(m_MainOutput.outputID == 0)
-  {
-    m_Width = width;
-    m_Height = height;
-  }
-}
-
 ReplayOutput::~ReplayOutput()
 {
   CHECK_REPLAY_THREAD();
@@ -146,6 +135,31 @@ void ReplayOutput::Shutdown()
   m_pRenderer->ShutdownOutput(this);
 }
 
+void ReplayOutput::SetDimensions(int32_t width, int32_t height)
+{
+  CHECK_REPLAY_THREAD();
+
+  m_pDevice->SetOutputWindowDimensions(m_MainOutput.outputID, width > 0 ? width : 1,
+                                       height > 0 ? height : 1);
+  m_pDevice->GetOutputWindowDimensions(m_MainOutput.outputID, m_Width, m_Height);
+}
+
+bytebuf ReplayOutput::ReadbackOutputTexture()
+{
+  CHECK_REPLAY_THREAD();
+
+  bytebuf data;
+  m_pDevice->GetOutputWindowData(m_MainOutput.outputID, data);
+  return data;
+}
+
+rdcpair<int32_t, int32_t> ReplayOutput::GetDimensions()
+{
+  CHECK_REPLAY_THREAD();
+
+  return make_rdcpair(m_Width, m_Height);
+}
+
 void ReplayOutput::SetTextureDisplay(const TextureDisplay &o)
 {
   CHECK_REPLAY_THREAD();
@@ -300,7 +314,7 @@ bool ReplayOutput::AddThumbnail(WindowingData window, ResourceId texID, CompType
 
   OutputPair p;
 
-  RDCASSERT(window.system != WindowingSystem::Unknown);
+  RDCASSERT(window.system != WindowingSystem::Unknown && window.system != WindowingSystem::Headless);
 
   bool depthMode = false;
 

util/test/tests/D3D11/D3D11_Overlay_Test.py

@@ -11,12 +11,10 @@ class D3D11_Overlay_Test(rdtest.TestCase):
     def check_capture(self):
         self.check_final_backbuffer()
 
-        out: rd.ReplayOutput = self.controller.CreateOutput(rd.CreateHeadlessWindowingData(), rd.ReplayOutputType.Texture)
+        out: rd.ReplayOutput = self.controller.CreateOutput(rd.CreateHeadlessWindowingData(100, 100), rd.ReplayOutputType.Texture)
 
         self.check(out is not None)
 
-        out.SetDimensions(100, 100)
-
         test_marker: rd.DrawcallDescription = self.find_draw("Test")
 
         self.controller.SetFrameEvent(test_marker.next.eventId, True)

util/test/tests/D3D12/D3D12_Overlay_Test.py

@@ -12,12 +12,10 @@ class D3D12_Overlay_Test(rdtest.TestCase):
     def check_capture(self):
         self.check_final_backbuffer()
 
-        out: rd.ReplayOutput = self.controller.CreateOutput(rd.CreateHeadlessWindowingData(), rd.ReplayOutputType.Texture)
+        out: rd.ReplayOutput = self.controller.CreateOutput(rd.CreateHeadlessWindowingData(100, 100), rd.ReplayOutputType.Texture)
 
         self.check(out is not None)
 
-        out.SetDimensions(100, 100)
-
         test_marker: rd.DrawcallDescription = self.find_draw("Test")
 
         self.controller.SetFrameEvent(test_marker.next.eventId, True)

util/test/tests/GL/GL_Overlay_Test.py

@@ -9,12 +9,10 @@ class GL_Overlay_Test(rdtest.TestCase):
     def check_capture(self):
         self.check_final_backbuffer()
 
-        out: rd.ReplayOutput = self.controller.CreateOutput(rd.CreateHeadlessWindowingData(), rd.ReplayOutputType.Texture)
+        out: rd.ReplayOutput = self.controller.CreateOutput(rd.CreateHeadlessWindowingData(100, 100), rd.ReplayOutputType.Texture)
 
         self.check(out is not None)
 
-        out.SetDimensions(100, 100)
-
         test_marker: rd.DrawcallDescription = self.find_draw("Test")
 
         self.controller.SetFrameEvent(test_marker.next.eventId, True)

util/test/tests/Vulkan/VK_Overlay_Test.py

@@ -9,12 +9,10 @@ class VK_Overlay_Test(rdtest.TestCase):
     def check_capture(self):
         self.check_final_backbuffer()
 
-        out: rd.ReplayOutput = self.controller.CreateOutput(rd.CreateHeadlessWindowingData(), rd.ReplayOutputType.Texture)
+        out: rd.ReplayOutput = self.controller.CreateOutput(rd.CreateHeadlessWindowingData(100, 100), rd.ReplayOutputType.Texture)
 
         self.check(out is not None)
 
-        out.SetDimensions(100, 100)
-
         test_marker: rd.DrawcallDescription = self.find_draw("Test")
 
         self.controller.SetFrameEvent(test_marker.next.eventId, True)