Do shader debugging without replaying the capture at all

* This requires that every operation on output windows leaves the state
  untouched, so make a few changes to arrange that.
* By doing this it means any unstable/non-deterministic results are
  consistent between the current capture inspection and the shader
  debug results.

renderdoc/driver/d3d11/d3d11_analyse.cpp

@@ -923,7 +923,7 @@ ShaderDebugTrace D3D11DebugManager::DebugVertex(uint32_t eventID, uint32_t verti
 
   ShaderDebugTrace empty;
 
-  m_WrappedDevice->ReplayLog(0, eventID, eReplay_WithoutDraw);
+  D3D11RenderStateTracker tracker(m_WrappedContext);
 
   ID3D11VertexShader *stateVS = NULL;
   m_WrappedContext->VSGetShader(&stateVS, NULL, NULL);
@@ -1266,7 +1266,7 @@ ShaderDebugTrace D3D11DebugManager::DebugPixel(uint32_t eventID, uint32_t x, uin
 
   ShaderDebugTrace empty;
 
-  m_WrappedDevice->ReplayLog(0, eventID, eReplay_WithoutDraw);
+  D3D11RenderStateTracker tracker(m_WrappedContext);
 
   ID3D11PixelShader *statePS = NULL;
   m_WrappedContext->PSGetShader(&statePS, NULL, NULL);
@@ -1799,9 +1799,6 @@ ShaderDebugTrace D3D11DebugManager::DebugPixel(uint32_t eventID, uint32_t x, uin
     return empty;
   }
 
-  // replay back to where we were, so we don't pick up modifications by this event in UAVs
-  m_WrappedDevice->ReplayLog(0, eventID, eReplay_WithoutDraw);
-
   ShaderDebugTrace traces[4];
 
   GlobalState global;
@@ -2183,7 +2180,7 @@ ShaderDebugTrace D3D11DebugManager::DebugThread(uint32_t eventID, uint32_t group
 
   ShaderDebugTrace empty;
 
-  m_WrappedDevice->ReplayLog(0, eventID, eReplay_WithoutDraw);
+  D3D11RenderStateTracker tracker(m_WrappedContext);
 
   ID3D11ComputeShader *stateCS = NULL;
   m_WrappedContext->CSGetShader(&stateCS, NULL, NULL);
@@ -2654,6 +2651,8 @@ void D3D11DebugManager::PickPixel(ResourceId texture, uint32_t x, uint32_t y, ui
                                   uint32_t mip, uint32_t sample, FormatComponentType typeHint,
                                   float pixel[4])
 {
+  D3D11RenderStateTracker tracker(m_WrappedContext);
+
   m_pImmediateContext->OMSetRenderTargets(1, &m_DebugRender.PickPixelRT, NULL);
 
   float color[4] = {0.0f, 0.0f, 0.0f, 0.0f};
@@ -2745,6 +2744,8 @@ void D3D11DebugManager::PickPixel(ResourceId texture, uint32_t x, uint32_t y, ui
 byte *D3D11DebugManager::GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mip,
                                         const GetTextureDataParams &params, size_t &dataSize)
 {
+  D3D11RenderStateTracker tracker(m_WrappedContext);
+
   ID3D11Resource *dummyTex = NULL;
 
   uint32_t subresource = 0;
@@ -3242,6 +3243,8 @@ ResourceId D3D11DebugManager::ApplyCustomShader(ResourceId shader, ResourceId te
 
   CreateCustomShaderTex(details.texWidth, details.texHeight);
 
+  D3D11RenderStateTracker tracker(m_WrappedContext);
+
   {
     D3D11_RENDER_TARGET_VIEW_DESC desc;
 
@@ -3368,7 +3371,7 @@ ResourceId D3D11DebugManager::RenderOverlay(ResourceId texid, FormatComponentTyp
     realTexDesc.SampleDesc.Quality = details.sampleQuality;
   }
 
-  D3D11RenderState old = *m_WrappedContext->GetCurrentPipelineState();
+  D3D11RenderStateTracker tracker(m_WrappedContext);
 
   D3D11_TEXTURE2D_DESC customTexDesc;
   RDCEraseEl(customTexDesc);
@@ -4519,8 +4522,6 @@ ResourceId D3D11DebugManager::RenderOverlay(ResourceId texid, FormatComponentTyp
   SAFE_RELEASE(renderDepth);
   SAFE_RELEASE(preDrawDepth);
 
-  old.ApplyState(m_WrappedContext);
-
   return m_OverlayResourceId;
 }
 

renderdoc/driver/d3d11/d3d11_debug.cpp

@@ -1679,6 +1679,12 @@ void D3D11DebugManager::BindOutputWindow(uint64_t id, bool depth)
   if(id == 0 || m_OutputWindows.find(id) == m_OutputWindows.end())
     return;
 
+  if(m_RealState.active)
+    RDCERR("Trashing RealState! Mismatched use of BindOutputWindow / FlipOutputWindow");
+
+  m_RealState.active = true;
+  m_RealState.state = *m_WrappedContext->GetCurrentPipelineState();
+
   m_WrappedContext->OMSetRenderTargets(
       1, &m_OutputWindows[id].rtv, depth && m_OutputWindows[id].dsv ? m_OutputWindows[id].dsv : NULL);
 
@@ -1704,6 +1710,17 @@ void D3D11DebugManager::FlipOutputWindow(uint64_t id)
 
   if(m_OutputWindows[id].swap)
     m_OutputWindows[id].swap->Present(0, 0);
+
+  if(m_RealState.active)
+  {
+    m_RealState.active = false;
+    m_RealState.state.ApplyState(m_WrappedContext);
+    m_RealState.state.Clear();
+  }
+  else
+  {
+    RDCERR("RealState wasn't active! Mismatched use of BindOutputWindow / FlipOutputWindow");
+  }
 }
 
 uint32_t D3D11DebugManager::GetStructCount(ID3D11UnorderedAccessView *uav)

renderdoc/driver/d3d11/d3d11_debug.h

@@ -31,6 +31,7 @@
 #include "api/replay/renderdoc_replay.h"
 #include "driver/dx/official/d3d11_4.h"
 #include "driver/shaders/dxbc/dxbc_debug.h"
+#include "d3d11_renderstate.h"
 
 using std::map;
 using std::pair;
@@ -316,6 +317,15 @@ private:
   uint64_t m_OutputWindowID;
   map<uint64_t, OutputWindow> m_OutputWindows;
 
+  // used to track the real state so we can preserve it even
+  // across work done to the output windows
+  struct RealState
+  {
+    RealState() : state((Serialiser *)NULL) { active = false; }
+    bool active;
+    D3D11RenderState state;
+  } m_RealState;
+
   static const uint32_t m_ShaderCacheMagic = 0xf000baba;
   static const uint32_t m_ShaderCacheVersion = 3;