From cd19c52aa5f6c5e5e2a7b6b0555bd912de27787f Mon Sep 17 00:00:00 2001
From: Jovan Ristic <jristic42@gmail.com>
Date: Tue, 12 Mar 2024 20:42:13 -0700
Subject: [PATCH] D3D12 Pixel History fix lack of integer clamping.

* When the shader outputs a value larger than the texture format
supports, the use of a 32bit target for history targets resulted in the
Tex After for fragments showing a value larger than possible.
* The int/uint path for ConvertAndFillInColor was missing the clamping
step that the other path includes.
---
 renderdoc/driver/d3d12/d3d12_pixelhistory.cpp | 22 +++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/renderdoc/driver/d3d12/d3d12_pixelhistory.cpp b/renderdoc/driver/d3d12/d3d12_pixelhistory.cpp
index 8d2499537..2330ecfc6 100644
--- a/renderdoc/driver/d3d12/d3d12_pixelhistory.cpp
+++ b/renderdoc/driver/d3d12/d3d12_pixelhistory.cpp
@@ -3013,6 +3013,28 @@ void ConvertAndFillInColor(ResourceFormat srcFmt, ResourceFormat outFmt,
   if((outFmt.compType == CompType::UInt) || (outFmt.compType == CompType::SInt))
   {
     PixelHistoryDecode(srcFmt, value.color, mod.col);
+    // Clamp values based on format
+    if(outFmt.compType == CompType::UInt)
+    {
+      uint32_t limits[4] = {
+          255,
+          UINT16_MAX,
+          0,
+          UINT32_MAX,
+      };
+      int limit_idx = outFmt.compByteWidth - 1;
+      for(size_t c = 0; c < outFmt.compCount; c++)
+        mod.col.uintValue[c] = RDCMIN(limits[limit_idx], mod.col.uintValue[c]);
+    }
+    else
+    {
+      int32_t limits[8] = {
+          INT8_MIN, INT8_MAX, INT16_MIN, INT16_MAX, 0, 0, INT32_MIN, INT32_MAX,
+      };
+      int limit_idx = 2 * (outFmt.compByteWidth - 1);
+      for(size_t c = 0; c < outFmt.compCount; c++)
+        mod.col.intValue[c] = RDCCLAMP(mod.col.intValue[c], limits[limit_idx], limits[limit_idx + 1]);
+    }
   }
   else
   {