From c21b64f84a390bd6301b99e1c04e97e353dd65dc Mon Sep 17 00:00:00 2001
From: baldurk <baldurk@baldurk.org>
Date: Wed, 16 Nov 2016 12:32:38 +0100
Subject: [PATCH] Fix stencil texture display

---
 renderdoc/driver/d3d12/d3d12_debug.cpp | 48 ++++++++++++++++++++------
 renderdoc/driver/dxgi/dxgi_common.cpp  | 18 ++++++++++
 renderdoc/driver/dxgi/dxgi_common.h    |  1 +
 3 files changed, 57 insertions(+), 10 deletions(-)

diff --git a/renderdoc/driver/d3d12/d3d12_debug.cpp b/renderdoc/driver/d3d12/d3d12_debug.cpp
index 24d684621..1e0a95a18 100644
--- a/renderdoc/driver/d3d12/d3d12_debug.cpp
+++ b/renderdoc/driver/d3d12/d3d12_debug.cpp
@@ -2326,6 +2326,9 @@ bool D3D12DebugManager::RenderTextureInternal(D3D12_CPU_DESCRIPTOR_HANDLE rtv, T
       srvOffset = RESTYPE_TEX2D_MS;
       srvDesc.ViewDimension = D3D12_SRV_DIMENSION_TEXTURE2DMSARRAY;
       srvDesc.Texture2DMSArray.ArraySize = ~0U;
+
+      if(IsDepthFormat(resourceDesc.Format))
+        srvOffset = RESTYPE_DEPTH_MS;
     }
     else
     {
@@ -2333,6 +2336,9 @@ bool D3D12DebugManager::RenderTextureInternal(D3D12_CPU_DESCRIPTOR_HANDLE rtv, T
       srvDesc.ViewDimension = D3D12_SRV_DIMENSION_TEXTURE2DARRAY;
       srvDesc.Texture2D.MipLevels = ~0U;
       srvDesc.Texture2DArray.ArraySize = ~0U;
+
+      if(IsDepthFormat(resourceDesc.Format))
+        srvOffset = RESTYPE_DEPTH;
     }
   }
   else if(resourceDesc.Dimension == D3D12_RESOURCE_DIMENSION_TEXTURE1D)
@@ -2345,6 +2351,11 @@ bool D3D12DebugManager::RenderTextureInternal(D3D12_CPU_DESCRIPTOR_HANDLE rtv, T
 
   pixelData.OutputDisplayFormat = srvOffset;
 
+  // if it's a depth and stencil image, increment (as the restype for
+  // depth/stencil is one higher than that for depth only).
+  if(IsDepthAndStencilFormat(resourceDesc.Format))
+    pixelData.OutputDisplayFormat++;
+
   if(cfg.overlay == eTexOverlay_NaN)
   {
     pixelData.OutputDisplayFormat |= TEXDISPLAY_NANS;
@@ -2379,7 +2390,8 @@ bool D3D12DebugManager::RenderTextureInternal(D3D12_CPU_DESCRIPTOR_HANDLE rtv, T
   D3D12_SHADER_RESOURCE_VIEW_DESC stencilSRVDesc = {};
 
   // for non-typeless depth formats, we need to copy to a typeless resource for read
-  if(IsDepthFormat(srvDesc.Format) && !IsTypelessFormat(srvDesc.Format))
+  if(IsDepthFormat(resourceDesc.Format) &&
+     GetTypelessFormat(resourceDesc.Format) != resourceDesc.Format)
   {
     realResourceState = D3D12_RESOURCE_STATE_COPY_SOURCE;
     copy = true;
@@ -2403,20 +2415,36 @@ bool D3D12DebugManager::RenderTextureInternal(D3D12_CPU_DESCRIPTOR_HANDLE rtv, T
                GetTypelessFormat(srvDesc.Format), srvDesc.Format);
         break;
     }
+  }
 
-    if(stencilSRVDesc.Format != DXGI_FORMAT_UNKNOWN)
+  // even for non-copies, we need to make two SRVs to sample stencil as well
+  if(IsDepthAndStencilFormat(resourceDesc.Format) && stencilSRVDesc.Format == DXGI_FORMAT_UNKNOWN)
+  {
+    switch(GetTypelessFormat(srvDesc.Format))
     {
-      D3D12_FEATURE_DATA_FORMAT_INFO formatInfo = {};
-      formatInfo.Format = srvDesc.Format;
-      m_WrappedDevice->CheckFeatureSupport(D3D12_FEATURE_FORMAT_INFO, &formatInfo,
-                                           sizeof(formatInfo));
-
-      if(formatInfo.PlaneCount > 1 &&
-         stencilSRVDesc.ViewDimension == D3D12_SRV_DIMENSION_TEXTURE2DARRAY)
-        stencilSRVDesc.Texture2DArray.PlaneSlice = 1;
+      case DXGI_FORMAT_R32G8X24_TYPELESS:
+        srvDesc.Format = DXGI_FORMAT_R32_FLOAT_X8X24_TYPELESS;
+        stencilSRVDesc = srvDesc;
+        stencilSRVDesc.Format = DXGI_FORMAT_X32_TYPELESS_G8X24_UINT;
+        break;
+      case DXGI_FORMAT_R24G8_TYPELESS:
+        srvDesc.Format = DXGI_FORMAT_R24_UNORM_X8_TYPELESS;
+        stencilSRVDesc = srvDesc;
+        stencilSRVDesc.Format = DXGI_FORMAT_X24_TYPELESS_G8_UINT;
+        break;
     }
   }
 
+  if(stencilSRVDesc.Format != DXGI_FORMAT_UNKNOWN)
+  {
+    D3D12_FEATURE_DATA_FORMAT_INFO formatInfo = {};
+    formatInfo.Format = srvDesc.Format;
+    m_WrappedDevice->CheckFeatureSupport(D3D12_FEATURE_FORMAT_INFO, &formatInfo, sizeof(formatInfo));
+
+    if(formatInfo.PlaneCount > 1 && stencilSRVDesc.ViewDimension == D3D12_SRV_DIMENSION_TEXTURE2DARRAY)
+      stencilSRVDesc.Texture2DArray.PlaneSlice = 1;
+  }
+
   FillBuffer(m_GenericVSCbuffer, &vertexData, sizeof(DebugVertexCBuffer));
   FillBuffer(m_GenericPSCbuffer, &pixelData, sizeof(DebugPixelCBufferData));
 
diff --git a/renderdoc/driver/dxgi/dxgi_common.cpp b/renderdoc/driver/dxgi/dxgi_common.cpp
index 9a6853a26..ec6c55c94 100644
--- a/renderdoc/driver/dxgi/dxgi_common.cpp
+++ b/renderdoc/driver/dxgi/dxgi_common.cpp
@@ -369,6 +369,24 @@ bool IsDepthFormat(DXGI_FORMAT f)
   return false;
 }
 
+bool IsDepthAndStencilFormat(DXGI_FORMAT f)
+{
+  switch(f)
+  {
+    case DXGI_FORMAT_R32G8X24_TYPELESS:
+    case DXGI_FORMAT_D32_FLOAT_S8X24_UINT:
+    case DXGI_FORMAT_R32_FLOAT_X8X24_TYPELESS:
+    case DXGI_FORMAT_X32_TYPELESS_G8X24_UINT:
+
+    case DXGI_FORMAT_D24_UNORM_S8_UINT:
+    case DXGI_FORMAT_R24_UNORM_X8_TYPELESS:
+    case DXGI_FORMAT_X24_TYPELESS_G8_UINT:
+    case DXGI_FORMAT_R24G8_TYPELESS: return true;
+  }
+
+  return false;
+}
+
 bool IsTypelessFormat(DXGI_FORMAT f)
 {
   switch(f)
diff --git a/renderdoc/driver/dxgi/dxgi_common.h b/renderdoc/driver/dxgi/dxgi_common.h
index 6c7082598..58cf68dd8 100644
--- a/renderdoc/driver/dxgi/dxgi_common.h
+++ b/renderdoc/driver/dxgi/dxgi_common.h
@@ -49,6 +49,7 @@ DXGI_FORMAT GetSRGBFormat(DXGI_FORMAT f);
 DXGI_FORMAT GetNonSRGBFormat(DXGI_FORMAT f);
 bool IsBlockFormat(DXGI_FORMAT f);
 bool IsDepthFormat(DXGI_FORMAT f);
+bool IsDepthAndStencilFormat(DXGI_FORMAT f);
 
 bool IsUIntFormat(DXGI_FORMAT f);
 bool IsTypelessFormat(DXGI_FORMAT f);