From d4bc4e29990826116379d83439ec494d5412c1e7 Mon Sep 17 00:00:00 2001
From: baldurk <baldurk@baldurk.org>
Date: Thu, 27 Nov 2014 00:49:05 +0000
Subject: [PATCH] Fix R11G11B10 decode - it's a truncated half format

* ie. not just an unnormalised mantissa plus biased exponent, it has proper
  IEEE style float properties with hidden bit on the mantissa etc. So full
  decoding is necessary, contrary to what some of the docs say :(.
---
 renderdoc/maths/formatpacking.h | 67 ++++++++++++++++++++++++++++-----
 1 file changed, 58 insertions(+), 9 deletions(-)

diff --git a/renderdoc/maths/formatpacking.h b/renderdoc/maths/formatpacking.h
index c79913849..56989004f 100644
--- a/renderdoc/maths/formatpacking.h
+++ b/renderdoc/maths/formatpacking.h
@@ -53,16 +53,65 @@ inline uint32_t ConvertToR10G10B10A2(Vec4f data)
 
 inline Vec3f ConvertFromR11G11B10(uint32_t data)
 {
-	uint32_t xMantissa = ((data>> 0) & 0x3f);
-	uint32_t xExponent = ((data>> 6) & 0x1f);
-	uint32_t yMantissa = ((data>>11) & 0x3f);
-	uint32_t yExponent = ((data>>17) & 0x1f);
-	uint32_t zMantissa = ((data>>22) & 0x1f);
-	uint32_t zExponent = ((data>>27) & 0x1f);
+	uint32_t mantissas[3] = {
+		(data>> 0) & 0x3f,
+		(data>>11) & 0x3f,
+		(data>>22) & 0x1f,
+	};
+	int32_t exponents[3] = {
+		(data>> 6) & 0x1f,
+		(data>>17) & 0x1f,
+		(data>>27) & 0x1f,
+	};
 
-	return Vec3f((float(xMantissa)/64.0f)*powf(2.0f, (float)xExponent - 15.0f),
-				 (float(yMantissa)/32.0f)*powf(2.0f, (float)yExponent - 15.0f),
-				 (float(zMantissa)/32.0f)*powf(2.0f, (float)zExponent - 15.0f));
+	Vec3f ret;
+	uint32_t *retu = (uint32_t *)&ret.x;
+
+	// floats have 23 bit mantissa, 8bit exponent
+	// R11G11B10 has 6/6/5 bit mantissas, 5bit exponents
+
+	const int mantissaShift[] = { 23-6, 23-6, 23-5 };
+
+	for(int i=0; i < 3; i++)
+	{
+		if(mantissas[i] == 0 && exponents[i] == 0)
+		{
+			retu[i] = 0;
+		}
+		else
+		{
+			if(exponents[i] == 0x1f)
+			{
+				// infinity or nan
+				retu[i] = 0x7f800000 | mantissas[i] << mantissaShift[i];
+			}
+			else if(exponents[i] != 0)
+			{
+				// shift exponent and mantissa to the right range for 32bit floats
+				retu[i] = (exponents[i] + (127-15)) << 23 | mantissas[i] << mantissaShift[i];
+			}
+			else if(exponents[i] == 0)
+			{
+				// we know xMantissa isn't 0 also, or it would have been caught above
+
+				exponents[i] = 1;
+
+				// shift until hidden bit is set
+				while((mantissas[i] & 0x40) == 0)
+				{
+					mantissas[i] <<= 1;
+					exponents[i]--;
+				}
+
+				// remove the hidden bit
+				mantissas[i] &= ~0x40;
+
+				retu[i] = (exponents[i] + (127-15)) << 23 | mantissas[i] << mantissaShift[i];
+			}
+		}
+	}
+
+	return ret;
 }
 
 /*