From 8e267c01ba7cac1776676444984c6e35bd456286 Mon Sep 17 00:00:00 2001 From: sindney <346567691@qq.com> Date: Tue, 16 Jun 2020 16:26:45 +0800 Subject: [PATCH] fix unpackData bug when unpacking UNorm and SNorm data --- docs/python_api/examples/renderdoc/decode_mesh.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/python_api/examples/renderdoc/decode_mesh.rst b/docs/python_api/examples/renderdoc/decode_mesh.rst index 9bba1e653..9b31d5d6c 100644 --- a/docs/python_api/examples/renderdoc/decode_mesh.rst +++ b/docs/python_api/examples/renderdoc/decode_mesh.rst @@ -115,10 +115,10 @@ For normalised formats - :py:attr:`~renderdoc.CompType.UNorm` and :py:attr:`~ren # If the format needs post-processing such as normalisation, do that now if fmt.compType == rd.CompType.UNorm: - divisor = float((1 << fmt.compByteWidth) - 1) + divisor = float((2 ** (fmt.compByteWidth * 8)) - 1) value = tuple(float(i) / divisor for i in value) elif fmt.compType == rd.CompType.SNorm: - maxNeg = -(1 << (fmt.compByteWidth - 1)) + maxNeg = -float((2 ** (fmt.compByteWidth * 8)) - 1) / 2 divisor = float(-(maxNeg-1)) value = tuple((float(i) if (i == maxNeg) else (float(i) / divisor)) for i in value)