From 982273bbd3f5ce98ce25c2be7f8b6c97feaf41ec Mon Sep 17 00:00:00 2001
From: baldurk <baldurk@baldurk.org>
Date: Mon, 11 Jul 2016 12:32:49 +0200
Subject: [PATCH] Detect and show sample mask failures in pixel history. Refs
 #296

---
 renderdoc/api/replay/data_types.h        |  1 +
 renderdoc/core/replay_proxy.cpp          |  3 ++-
 renderdoc/driver/d3d11/d3d11_analyse.cpp | 12 ++++++++++++
 renderdocui/Interop/FetchInfo.cs         |  7 ++++---
 renderdocui/Windows/PixelHistoryView.cs  |  2 ++
 5 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/renderdoc/api/replay/data_types.h b/renderdoc/api/replay/data_types.h
index 3f98b902d..b4ccd2520 100644
--- a/renderdoc/api/replay/data_types.h
+++ b/renderdoc/api/replay/data_types.h
@@ -483,6 +483,7 @@ struct PixelModification
   ModificationValue shaderOut;
   ModificationValue postMod;
 
+  bool32 sampleMasked;
   bool32 backfaceCulled;
   bool32 depthClipped;
   bool32 viewClipped;
diff --git a/renderdoc/core/replay_proxy.cpp b/renderdoc/core/replay_proxy.cpp
index 99df15d30..95c5dd5be 100644
--- a/renderdoc/core/replay_proxy.cpp
+++ b/renderdoc/core/replay_proxy.cpp
@@ -1398,6 +1398,7 @@ void Serialiser::Serialise(const char *name, PixelModification &el)
   Serialise("", el.postMod.depth);
   Serialise("", el.postMod.stencil);
 
+  Serialise("", el.sampleMasked);
   Serialise("", el.backfaceCulled);
   Serialise("", el.depthClipped);
   Serialise("", el.viewClipped);
@@ -1406,7 +1407,7 @@ void Serialiser::Serialise(const char *name, PixelModification &el)
   Serialise("", el.depthTestFailed);
   Serialise("", el.stencilTestFailed);
 
-  SIZE_CHECK(PixelModification, 116);
+  SIZE_CHECK(PixelModification, 120);
 }
 
 #pragma endregion Data descriptors
diff --git a/renderdoc/driver/d3d11/d3d11_analyse.cpp b/renderdoc/driver/d3d11/d3d11_analyse.cpp
index 0f814cac6..fca59680f 100644
--- a/renderdoc/driver/d3d11/d3d11_analyse.cpp
+++ b/renderdoc/driver/d3d11/d3d11_analyse.cpp
@@ -4623,6 +4623,9 @@ vector<PixelModification> D3D11DebugManager::PixelHistory(vector<EventUsage> eve
     TestMustFail_DepthTesting = 1 << 8,      // if the comparison func is NEVER
     TestMustFail_StencilTesting = 1 << 9,    // if the comparison func is NEVER for both faces, or
                                              // one face is backface culled and the other is NEVER
+
+    // if the sample mask set at this event doesn't have the right bit set
+    TestMustFail_SampleMask = 1 << 10,
   };
 
 #if 1
@@ -4936,6 +4939,13 @@ vector<PixelModification> D3D11DebugManager::PixelHistory(vector<EventUsage> eve
       // no blending enabled by default
     }
 
+    // sampleMask is a mask containing only the bit for the sample we want
+    // (or 0xFFFFFFFF if no sample was chosen and we are looking at them all).
+    if((curSample & sampleMask) == 0)
+    {
+      flags[ev] |= TestMustFail_SampleMask;
+    }
+
     m_pDevice->CreateRasterizerState(&rd, &newRS);
     m_pImmediateContext->RSSetState(newRS);
     SAFE_RELEASE(newRS);
@@ -5309,6 +5319,8 @@ vector<PixelModification> D3D11DebugManager::PixelHistory(vector<EventUsage> eve
           mod.stencilTestFailed = true;
         if(flags[i] & TestMustFail_Scissor)
           mod.scissorClipped = true;
+        if(flags[i] & TestMustFail_SampleMask)
+          mod.sampleMasked = true;
 
         m_WrappedDevice->ReplayLog(0, events[i].eventID, eReplay_WithoutDraw);
 
diff --git a/renderdocui/Interop/FetchInfo.cs b/renderdocui/Interop/FetchInfo.cs
index 3d067bec6..380c0c634 100644
--- a/renderdocui/Interop/FetchInfo.cs
+++ b/renderdocui/Interop/FetchInfo.cs
@@ -814,6 +814,7 @@ namespace renderdoc
         [CustomMarshalAs(CustomUnmanagedType.CustomClass)]
         public ModificationValue postMod;
 
+        public bool sampleMasked;
         public bool backfaceCulled;
         public bool depthClipped;
         public bool viewClipped;
@@ -824,9 +825,9 @@ namespace renderdoc
 
         public bool EventPassed()
         {
-            return !backfaceCulled && !depthClipped && !viewClipped &&
-                !scissorClipped && !shaderDiscarded && !depthTestFailed &&
-                !stencilTestFailed;
+            return !sampleMasked && !backfaceCulled && !depthClipped &&
+                !viewClipped && !scissorClipped && !shaderDiscarded &&
+                !depthTestFailed && !stencilTestFailed;
         }
     };
 }
diff --git a/renderdocui/Windows/PixelHistoryView.cs b/renderdocui/Windows/PixelHistoryView.cs
index 7dffb8d01..68374680d 100644
--- a/renderdocui/Windows/PixelHistoryView.cs
+++ b/renderdocui/Windows/PixelHistoryView.cs
@@ -233,6 +233,8 @@ namespace renderdocui.Windows
         {
             string s = "";
 
+            if (mod.sampleMasked)
+                s += "\nMasked by SampleMask";
             if (mod.backfaceCulled)
                 s += "\nBackface culled";
             if (mod.depthClipped)