From f62f281415d81bee8728b7530335ce01e707d801 Mon Sep 17 00:00:00 2001
From: baldurk <baldurk@baldurk.org>
Date: Wed, 11 Oct 2017 11:11:52 +0100
Subject: [PATCH] Fix handling of D3D11_KEEP_{RTs/UAVs} in some edge cases

---
 renderdoc/driver/d3d11/d3d11_context.cpp      | 48 +++++++++++--------
 renderdoc/driver/d3d11/d3d11_context_wrap.cpp | 21 ++++++--
 2 files changed, 45 insertions(+), 24 deletions(-)

diff --git a/renderdoc/driver/d3d11/d3d11_context.cpp b/renderdoc/driver/d3d11/d3d11_context.cpp
index c7baf02b4..9fdeb50f1 100644
--- a/renderdoc/driver/d3d11/d3d11_context.cpp
+++ b/renderdoc/driver/d3d11/d3d11_context.cpp
@@ -1969,38 +1969,48 @@ void WrappedID3D11DeviceContext::RecordOutputMergerStats(UINT NumRTVs, ID3D11Ren
   // #mivance is an elaborate redundancy here even useful?
   // const D3D11RenderState::outmerger* Current = &m_CurrentPipelineState->OM;
 
-  if(RTVs != NULL)
+  if(NumRTVs != D3D11_KEEP_RENDER_TARGETS_AND_DEPTH_STENCIL)
   {
-    for(UINT index = 0; index < NumRTVs; index++)
+    if(RTVs != NULL)
     {
-      outputs.sets += (RTVs[index] != NULL);
-      outputs.nulls += (RTVs[index] == NULL);
+      for(UINT index = 0; index < NumRTVs; index++)
+      {
+        outputs.sets += (RTVs[index] != NULL);
+        outputs.nulls += (RTVs[index] == NULL);
+      }
+    }
+    else
+    {
+      outputs.nulls += NumRTVs;
     }
-  }
-  else if(NumRTVs != D3D11_KEEP_RENDER_TARGETS_AND_DEPTH_STENCIL)
-  {
-    outputs.nulls += NumRTVs;
   }
 
   outputs.sets += (DSV != NULL);
   outputs.nulls += (DSV == NULL);
 
-  if(UAVs != NULL)
+  if(NumUAVs != D3D11_KEEP_UNORDERED_ACCESS_VIEWS)
   {
-    for(UINT index = 0; index < NumUAVs; index++)
+    if(UAVs != NULL)
     {
-      outputs.sets += (UAVs[index] != NULL);
-      outputs.nulls += (UAVs[index] == NULL);
+      for(UINT index = 0; index < NumUAVs; index++)
+      {
+        outputs.sets += (UAVs[index] != NULL);
+        outputs.nulls += (UAVs[index] == NULL);
+      }
+    }
+    else
+    {
+      outputs.nulls += NumUAVs;
     }
   }
-  else
-  {
-    outputs.nulls += NumUAVs;
-  }
 
-  UINT NumSlots = NumRTVs + NumUAVs;
-  if(NumRTVs == D3D11_KEEP_RENDER_TARGETS_AND_DEPTH_STENCIL)
-    NumSlots = NumUAVs;
+  UINT NumSlots = 0;
+
+  if(NumRTVs != D3D11_KEEP_RENDER_TARGETS_AND_DEPTH_STENCIL)
+    NumSlots += NumRTVs;
+  if(NumRTVs != D3D11_KEEP_UNORDERED_ACCESS_VIEWS)
+    NumSlots += NumUAVs;
+
   RDCASSERT(NumSlots < outputs.bindslots.size());
   outputs.bindslots[NumSlots] += 1;
 }
diff --git a/renderdoc/driver/d3d11/d3d11_context_wrap.cpp b/renderdoc/driver/d3d11/d3d11_context_wrap.cpp
index 206f557ba..91a700059 100644
--- a/renderdoc/driver/d3d11/d3d11_context_wrap.cpp
+++ b/renderdoc/driver/d3d11/d3d11_context_wrap.cpp
@@ -3392,7 +3392,12 @@ void WrappedID3D11DeviceContext::OMSetRenderTargetsAndUnorderedAccessViews(
   // D3D11 doesn't seem to complain about this case, but it messes our render state tracking so
   // ensure we don't blat over any RTs with 'empty' UAVs.
   if(NumUAVs == 0)
-    UAVStartSlot = RDCMAX(NumRTVs, UAVStartSlot);
+  {
+    if(NumRTVs != D3D11_KEEP_RENDER_TARGETS_AND_DEPTH_STENCIL)
+      UAVStartSlot = RDCMAX(NumRTVs, UAVStartSlot);
+    else
+      UAVStartSlot = RDCMAX(m_CurrentPipelineState->OM.UAVStartSlot, UAVStartSlot);
+  }
 
   if(m_State == WRITING_CAPFRAME)
   {
@@ -3408,10 +3413,13 @@ void WrappedID3D11DeviceContext::OMSetRenderTargetsAndUnorderedAccessViews(
   ID3D11RenderTargetView *RTs[D3D11_SIMULTANEOUS_RENDER_TARGET_COUNT] = {0};
   ID3D11UnorderedAccessView *UAVs[D3D11_1_UAV_SLOT_COUNT] = {0};
 
-  for(UINT i = 0; ppRenderTargetViews && i < NumRTVs; i++)
+  for(UINT i = 0;
+      ppRenderTargetViews && NumRTVs != D3D11_KEEP_RENDER_TARGETS_AND_DEPTH_STENCIL && i < NumRTVs;
+      i++)
     RTs[i] = ppRenderTargetViews[i];
 
-  for(UINT i = 0; ppUnorderedAccessViews && i < NumUAVs; i++)
+  for(UINT i = 0;
+      ppUnorderedAccessViews && NumUAVs != D3D11_KEEP_UNORDERED_ACCESS_VIEWS && i < NumUAVs; i++)
     UAVs[i] = ppUnorderedAccessViews[i];
 
   if(NumRTVs != D3D11_KEEP_RENDER_TARGETS_AND_DEPTH_STENCIL)
@@ -3478,7 +3486,9 @@ void WrappedID3D11DeviceContext::OMSetRenderTargetsAndUnorderedAccessViews(
                                            RDCMIN(NumUAVs, D3D11_1_UAV_SLOT_COUNT - StartSlot));
   }
 
-  for(UINT i = 0; ppRenderTargetViews && i < NumRTVs; i++)
+  for(UINT i = 0;
+      ppRenderTargetViews && NumRTVs != D3D11_KEEP_RENDER_TARGETS_AND_DEPTH_STENCIL && i < NumRTVs;
+      i++)
   {
     if(ppRenderTargetViews[i] && m_State >= WRITING)
     {
@@ -3494,7 +3504,8 @@ void WrappedID3D11DeviceContext::OMSetRenderTargetsAndUnorderedAccessViews(
     RTs[i] = UNWRAP(WrappedID3D11RenderTargetView1, ppRenderTargetViews[i]);
   }
 
-  for(UINT i = 0; ppUnorderedAccessViews && i < NumUAVs; i++)
+  for(UINT i = 0;
+      ppUnorderedAccessViews && NumUAVs != D3D11_KEEP_UNORDERED_ACCESS_VIEWS && i < NumUAVs; i++)
   {
     if(ppUnorderedAccessViews[i] && m_State >= WRITING)
     {