From 689f6960c0f44be14864585d3c058aff38631c6c Mon Sep 17 00:00:00 2001 From: sindney <346567691@qq.com> Date: Tue, 16 Jun 2020 16:32:32 +0800 Subject: [PATCH] fix unpackData max negative --- docs/python_api/examples/renderdoc/decode_mesh.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/python_api/examples/renderdoc/decode_mesh.rst b/docs/python_api/examples/renderdoc/decode_mesh.rst index 9b31d5d6c..fc5286d09 100644 --- a/docs/python_api/examples/renderdoc/decode_mesh.rst +++ b/docs/python_api/examples/renderdoc/decode_mesh.rst @@ -118,7 +118,7 @@ For normalised formats - :py:attr:`~renderdoc.CompType.UNorm` and :py:attr:`~ren divisor = float((2 ** (fmt.compByteWidth * 8)) - 1) value = tuple(float(i) / divisor for i in value) elif fmt.compType == rd.CompType.SNorm: - maxNeg = -float((2 ** (fmt.compByteWidth * 8)) - 1) / 2 + maxNeg = -float(2 ** (fmt.compByteWidth * 8)) / 2 divisor = float(-(maxNeg-1)) value = tuple((float(i) if (i == maxNeg) else (float(i) / divisor)) for i in value)