diff --git a/docs/python_api/examples/renderdoc/decode_mesh.rst b/docs/python_api/examples/renderdoc/decode_mesh.rst index 9bba1e653..9b31d5d6c 100644 --- a/docs/python_api/examples/renderdoc/decode_mesh.rst +++ b/docs/python_api/examples/renderdoc/decode_mesh.rst @@ -115,10 +115,10 @@ For normalised formats - :py:attr:`~renderdoc.CompType.UNorm` and :py:attr:`~ren # If the format needs post-processing such as normalisation, do that now if fmt.compType == rd.CompType.UNorm: - divisor = float((1 << fmt.compByteWidth) - 1) + divisor = float((2 ** (fmt.compByteWidth * 8)) - 1) value = tuple(float(i) / divisor for i in value) elif fmt.compType == rd.CompType.SNorm: - maxNeg = -(1 << (fmt.compByteWidth - 1)) + maxNeg = -float((2 ** (fmt.compByteWidth * 8)) - 1) / 2 divisor = float(-(maxNeg-1)) value = tuple((float(i) if (i == maxNeg) else (float(i) / divisor)) for i in value)