Improve ability to run a replay output with no backing window

* This is useful for scripts where we want access to the contextual output
  operations like overlays, pixel picking, etc.

renderdoc/api/replay/renderdoc_replay.h

@@ -311,6 +311,7 @@ DOCUMENT(R"(Specifies a windowing system to use for creating an output window.
 .. data:: Unknown
 
   No windowing data is passed and no native window is described.
+  See :func:`CreateHeadlessWindowingData`.
 
 .. data:: Win32
 
@@ -411,6 +412,20 @@ DECLARE_REFLECTION_ENUM(WindowingData);
 
 #endif
 
+DOCUMENT(R"(Create a :class:`WindowingData` for no backing window, it will be headless.
+
+:return: A :class:`WindowingData` corresponding to an 'empty' backing window.
+:rtype: WindowingData
+)");
+inline const WindowingData CreateHeadlessWindowingData()
+{
+  WindowingData ret = {};
+
+  ret.system = WindowingSystem::Unknown;
+
+  return ret;
+}
+
 DOCUMENT(R"(Create a :class:`WindowingData` for a Win32 ``HWND`` handle.
 
 :param HWND window: The native ``HWND`` handle for this window.
@@ -603,6 +618,17 @@ outputs.
   DOCUMENT("Sets the :class:`MeshDisplay` configuration for a mesh output.");
   virtual void SetMeshDisplay(const MeshDisplay &o) = 0;
 
+  DOCUMENT(R"(Sets the dimensions of the output, useful only for headless outputs that don't have a
+backing window which don't have any implicit dimensions. This allows configuring a virtual viewport
+which is useful for operations like picking vertices that depends on the output dimensions.
+
+.. note:: For outputs with backing windows, this will be ignored.
+
+:param int width: The width to use.
+:param int height: The height to use.
+)");
+  virtual void SetDimensions(int32_t width, int32_t height) = 0;
+
   DOCUMENT(
       "Clear and release all thumbnails associated with this output. See :meth:`AddThumbnail`.");
   virtual void ClearThumbnails() = 0;

renderdoc/replay/replay_controller.h

@@ -40,6 +40,7 @@ public:
   void Shutdown();
   void SetTextureDisplay(const TextureDisplay &o);
   void SetMeshDisplay(const MeshDisplay &o);
+  void SetDimensions(int32_t width, int32_t height);
 
   void ClearThumbnails();
   bool AddThumbnail(WindowingData window, ResourceId texID, CompType typeHint);
@@ -54,8 +55,8 @@ public:
   rdcpair<PixelValue, PixelValue> GetMinMax();
   rdcarray<uint32_t> GetHistogram(float minval, float maxval, bool channels[4]);
 
-  ResourceId GetCustomShaderTexID() { return m_CustomShaderResourceId; }
-  ResourceId GetDebugOverlayTexID() { return m_OverlayResourceId; }
+  ResourceId GetCustomShaderTexID();
+  ResourceId GetDebugOverlayTexID();
   PixelValue PickPixel(ResourceId texID, bool customShader, uint32_t x, uint32_t y,
                        uint32_t sliceFace, uint32_t mip, uint32_t sample);
   rdcpair<uint32_t, uint32_t> PickVertex(uint32_t eventId, uint32_t x, uint32_t y);

renderdoc/replay/replay_output.cpp

@@ -114,6 +114,15 @@ ReplayOutput::ReplayOutput(ReplayController *parent, WindowingData window, Repla
     RenderDoc::Inst().GetCrashHandler()->RegisterMemoryRegion(this, sizeof(ReplayController));
 }
 
+void ReplayOutput::SetDimensions(int32_t width, int32_t height)
+{
+  if(m_MainOutput.outputID == 0)
+  {
+    m_Width = width;
+    m_Height = height;
+  }
+}
+
 ReplayOutput::~ReplayOutput()
 {
   m_pDevice->DestroyOutputWindow(m_MainOutput.outputID);
@@ -225,6 +234,23 @@ void ReplayOutput::RefreshOverlay()
   }
 }
 
+ResourceId ReplayOutput::GetCustomShaderTexID()
+{
+  return m_CustomShaderResourceId;
+}
+
+ResourceId ReplayOutput::GetDebugOverlayTexID()
+{
+  if(m_OverlayDirty)
+  {
+    m_pDevice->ReplayLog(m_EventID, eReplay_WithoutDraw);
+    RefreshOverlay();
+    m_pDevice->ReplayLog(m_EventID, eReplay_OnlyDraw);
+  }
+
+  return m_OverlayResourceId;
+}
+
 void ReplayOutput::ClearThumbnails()
 {
   for(size_t i = 0; i < m_Thumbnails.size(); i++)