From 6f29b8b939623b123ec629446e53af234182672f Mon Sep 17 00:00:00 2001
From: baldurk <baldurk@baldurk.org>
Date: Thu, 27 Oct 2022 16:22:54 +0100
Subject: [PATCH] Work around fxc warnings when building with optimisation
 disabled

---
 renderdoc/data/hlsl/mesh.hlsl       | 48 ++++++++++++++++-------------
 renderdoc/data/hlsl/texdisplay.hlsl | 10 ++++--
 2 files changed, 35 insertions(+), 23 deletions(-)

diff --git a/renderdoc/data/hlsl/mesh.hlsl b/renderdoc/data/hlsl/mesh.hlsl
index b9d423c85..a6ed19db8 100644
--- a/renderdoc/data/hlsl/mesh.hlsl
+++ b/renderdoc/data/hlsl/mesh.hlsl
@@ -149,34 +149,40 @@ AppendStructuredBuffer<uint4> pickresult : register(u0);
 bool TriangleRayIntersect(float3 A, float3 B, float3 C, float3 RayPosition, float3 RayDirection,
                           out float3 HitPosition)
 {
+  HitPosition = RayPosition;
+
   bool Result = false;
 
   if(all(A == B) || all(A == C) || all(B == C))
-    return false;
-
-  float3 v0v1 = B - A;
-  float3 v0v2 = C - A;
-  float3 pvec = cross(RayDirection, v0v2);
-  float det = dot(v0v1, pvec);
-
-  // if the determinant is negative the triangle is backfacing, but we still take those!
-  // if the determinant is close to 0, the ray misses the triangle
-  if(abs(det) > 0)
   {
-    float invDet = 1 / det;
+    Result = false;
+  }
+  else
+  {
+    float3 v0v1 = B - A;
+    float3 v0v2 = C - A;
+    float3 pvec = cross(RayDirection, v0v2);
+    float det = dot(v0v1, pvec);
 
-    float3 tvec = RayPosition - A;
-    float3 qvec = cross(tvec, v0v1);
-    float u = dot(tvec, pvec) * invDet;
-    float v = dot(RayDirection, qvec) * invDet;
-
-    if(u >= 0 && u <= 1 && v >= 0 && u + v <= 1)
+    // if the determinant is negative the triangle is backfacing, but we still take those!
+    // if the determinant is close to 0, the ray misses the triangle
+    if(abs(det) > 0)
     {
-      float t = dot(v0v2, qvec) * invDet;
-      if(t >= 0)
+      float invDet = 1 / det;
+
+      float3 tvec = RayPosition - A;
+      float3 qvec = cross(tvec, v0v1);
+      float u = dot(tvec, pvec) * invDet;
+      float v = dot(RayDirection, qvec) * invDet;
+
+      if(u >= 0 && u <= 1 && v >= 0 && u + v <= 1)
       {
-        HitPosition = RayPosition + (RayDirection * t);
-        Result = true;
+        float t = dot(v0v2, qvec) * invDet;
+        if(t >= 0)
+        {
+          HitPosition = RayPosition + (RayDirection * t);
+          Result = true;
+        }
       }
     }
   }
diff --git a/renderdoc/data/hlsl/texdisplay.hlsl b/renderdoc/data/hlsl/texdisplay.hlsl
index c0d4ee442..8e2eb8fe0 100644
--- a/renderdoc/data/hlsl/texdisplay.hlsl
+++ b/renderdoc/data/hlsl/texdisplay.hlsl
@@ -55,6 +55,11 @@ v2f RENDERDOC_TexDisplayVS(uint vertID : SV_VertexID)
   return OUT;
 }
 
+bool fxc_workaround_isnan(float f)
+{
+  return (asuint(f) & 0x7fffffff) > 0x7f800000;
+}
+
 // main texture display shader, used for the texture viewer. It samples the right resource
 // for the type and applies things like the range check and channel masking.
 // It also does a couple of overlays that we can get 'free' like NaN/inf checks
@@ -168,6 +173,7 @@ float4 RENDERDOC_TexDisplayPS(v2f IN) : SV_Target0
   // workaround for D3DCompiler bug. For some reason it assumes texture samples can
   // never come back as NaN, so involving a cbuffer value like this here ensures
   // the below isnan()s don't get optimised out.
+  // update: this doesn't work when optimisation is disabled :(
   if(Channels.x < 0.5f)
     col.x = pre_range_col.x = AlwaysZero;
   if(Channels.y < 0.5f)
@@ -180,8 +186,8 @@ float4 RENDERDOC_TexDisplayPS(v2f IN) : SV_Target0
   // show nans, infs and negatives
   if(OutputDisplayFormat & TEXDISPLAY_NANS)
   {
-    if(isnan(pre_range_col.r) || isnan(pre_range_col.g) || isnan(pre_range_col.b) ||
-       isnan(pre_range_col.a))
+    if(fxc_workaround_isnan(pre_range_col.r) || fxc_workaround_isnan(pre_range_col.g) ||
+       fxc_workaround_isnan(pre_range_col.b) || fxc_workaround_isnan(pre_range_col.a))
       return float4(1, 0, 0, 1);
 
     if(isinf(pre_range_col.r) || isinf(pre_range_col.g) || isinf(pre_range_col.b) ||