lee/renderdoc
1
0
Fork 0
mirror of https://github.com/baldurk/renderdoc.git synced 2026-05-06 01:50:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactor ShaderVariable use and non-32-bit precision. Closes #2466

Browse Source
This commit is contained in:
baldurk
2022-02-02 11:38:37 +00:00
parent 4b748c6368
commit e42b0ff2ca
18 changed files with 508 additions and 351 deletions
Download Patch File Download Diff File Expand all files Collapse all files
qrenderdoc/Code/BufferFormatter.cpp
+96 -47
View File
@@ -1044,22 +1044,40 @@ static void FillShaderVarData(ShaderVariable &var, const ShaderConstant &elem, c
src++;
if(var.type == VarType::Double)
var.value.f64v[dst] = o.toDouble();
if(var.type == VarType::Float || var.type == VarType::Half)
var.value.f32v[dst] = o.toFloat();
else if(var.type == VarType::ULong)
var.value.u64v[dst] = o.toULongLong();
else if(var.type == VarType::SLong)
var.value.s64v[dst] = o.toLongLong();
else if(var.type == VarType::Bool)
var.value.u32v[dst] = o.toBool() ? 1 : 0;
else if(var.type == VarType::UInt || var.type == VarType::UShort || var.type == VarType::UByte)
var.value.u32v[dst] = o.toUInt();
else if(var.type == VarType::SInt || var.type == VarType::SShort || var.type == VarType::SByte)
var.value.s32v[dst] = o.toInt();
else
var.value.f32v[dst] = o.toFloat();
switch(var.type)
{
case VarType::Float: var.value.f32v[dst] = o.toFloat(); break;
case VarType::Double: var.value.f64v[dst] = o.toDouble(); break;
case VarType::Half: var.value.f16v[dst] = rdhalf::make(o.toFloat()); break;
case VarType::Bool: var.value.u32v[dst] = o.toBool() ? 1 : 0; break;
case VarType::ULong: var.value.u64v[dst] = o.toULongLong(); break;
case VarType::UInt: var.value.u32v[dst] = o.toUInt(); break;
case VarType::UShort: var.value.u16v[dst] = o.toUInt() & 0xffff; break;
case VarType::UByte: var.value.u8v[dst] = o.toUInt() & 0xff; break;
case VarType::SLong: var.value.s64v[dst] = o.toLongLong(); break;
case VarType::SInt:
var.value.s32v[dst] = o.toInt();
break;
break;
case VarType::SShort:
var.value.u16v[dst] = (int16_t)qBound((int)INT16_MIN, o.toInt(), (int)INT16_MAX);
break;
case VarType::SByte:
var.value.u8v[dst] = (int8_t)qBound((int)INT8_MIN, o.toInt(), (int)INT8_MAX);
break;
case VarType::GPUPointer:
// treat this as a 64-bit unsigned integer
var.value.u64v[dst] = o.toULongLong();
break;
case VarType::ConstantBlock:
case VarType::ReadOnlyResource:
case VarType::ReadWriteResource:
case VarType::Sampler:
case VarType::Unknown:
qCritical() << "Unexpected variable type" << ToQStr(var.type) << "in variable"
<< (QString)var.name;
break;
}
}
}
}
@@ -1469,7 +1487,7 @@ QVariantList GetVariants(ResourceFormat format, const ShaderConstantDescriptor &
else if(format.compByteWidth == 4)
ret.push_back(readObj<float>(data, end, ok));
else if(format.compByteWidth == 2)
ret.push_back(RENDERDOC_HalfToFloat(readObj<uint16_t>(data, end, ok)));
ret.push_back((float)rdhalf::make(readObj<uint16_t>(data, end, ok)));
}
else if(format.compType == CompType::SInt)
{
@@ -1638,36 +1656,67 @@ QString RowString(const ShaderVariable &v, uint32_t row, VarType type)
if(v.type == VarType::GPUPointer)
return ToQStr(v.GetPointer());
if(type == VarType::Double)
return RowValuesToString((int)v.columns, v.displayAsHex, v.value.f64v[row * v.columns + 0],
v.value.f64v[row * v.columns + 1], v.value.f64v[row * v.columns + 2],
v.value.f64v[row * v.columns + 3]);
else if(type == VarType::SLong)
return RowValuesToString((int)v.columns, v.displayAsHex, v.value.s64v[row * v.columns + 0],
v.value.s64v[row * v.columns + 1], v.value.s64v[row * v.columns + 2],
v.value.s64v[row * v.columns + 3]);
else if(type == VarType::ULong)
return RowValuesToString((int)v.columns, v.displayAsHex, v.value.u64v[row * v.columns + 0],
v.value.u64v[row * v.columns + 1], v.value.u64v[row * v.columns + 2],
v.value.u64v[row * v.columns + 3]);
else if(type == VarType::SInt || type == VarType::SShort || type == VarType::SByte)
return RowValuesToString((int)v.columns, v.displayAsHex, v.value.s32v[row * v.columns + 0],
v.value.s32v[row * v.columns + 1], v.value.s32v[row * v.columns + 2],
v.value.s32v[row * v.columns + 3]);
else if(type == VarType::UInt || type == VarType::UShort || type == VarType::UByte)
return RowValuesToString((int)v.columns, v.displayAsHex, v.value.u32v[row * v.columns + 0],
v.value.u32v[row * v.columns + 1], v.value.u32v[row * v.columns + 2],
v.value.u32v[row * v.columns + 3]);
else if(type == VarType::Bool)
return RowValuesToString((int)v.columns, v.displayAsHex,
v.value.u32v[row * v.columns + 0] ? true : false,
v.value.u32v[row * v.columns + 1] ? true : false,
v.value.u32v[row * v.columns + 2] ? true : false,
v.value.u32v[row * v.columns + 3] ? true : false);
else
return RowValuesToString((int)v.columns, v.displayAsHex, v.value.f32v[row * v.columns + 0],
v.value.f32v[row * v.columns + 1], v.value.f32v[row * v.columns + 2],
v.value.f32v[row * v.columns + 3]);
switch(type)
{
case VarType::Float:
return RowValuesToString((int)v.columns, v.displayAsHex, v.value.f32v[row * v.columns + 0],
v.value.f32v[row * v.columns + 1], v.value.f32v[row * v.columns + 2],
v.value.f32v[row * v.columns + 3]);
case VarType::Double:
return RowValuesToString((int)v.columns, v.displayAsHex, v.value.f64v[row * v.columns + 0],
v.value.f64v[row * v.columns + 1], v.value.f64v[row * v.columns + 2],
v.value.f64v[row * v.columns + 3]);
case VarType::Half:
return RowValuesToString((int)v.columns, v.displayAsHex, v.value.f16v[row * v.columns + 0],
v.value.f16v[row * v.columns + 1], v.value.f16v[row * v.columns + 2],
v.value.f16v[row * v.columns + 3]);
case VarType::Bool:
return RowValuesToString((int)v.columns, v.displayAsHex,
v.value.u32v[row * v.columns + 0] ? true : false,
v.value.u32v[row * v.columns + 1] ? true : false,
v.value.u32v[row * v.columns + 2] ? true : false,
v.value.u32v[row * v.columns + 3] ? true : false);
case VarType::ULong:
return RowValuesToString((int)v.columns, v.displayAsHex, v.value.u64v[row * v.columns + 0],
v.value.u64v[row * v.columns + 1], v.value.u64v[row * v.columns + 2],
v.value.u64v[row * v.columns + 3]);
case VarType::UInt:
return RowValuesToString((int)v.columns, v.displayAsHex, v.value.u32v[row * v.columns + 0],
v.value.u32v[row * v.columns + 1], v.value.u32v[row * v.columns + 2],
v.value.u32v[row * v.columns + 3]);
case VarType::UShort:
return RowValuesToString((int)v.columns, v.displayAsHex, v.value.u16v[row * v.columns + 0],
v.value.u16v[row * v.columns + 1], v.value.u16v[row * v.columns + 2],
v.value.u16v[row * v.columns + 3]);
case VarType::UByte:
return RowValuesToString((int)v.columns, v.displayAsHex, v.value.u8v[row * v.columns + 0],
v.value.u8v[row * v.columns + 1], v.value.u8v[row * v.columns + 2],
v.value.u8v[row * v.columns + 3]);
case VarType::SLong:
return RowValuesToString((int)v.columns, v.displayAsHex, v.value.s64v[row * v.columns + 0],
v.value.s64v[row * v.columns + 1], v.value.s64v[row * v.columns + 2],
v.value.s64v[row * v.columns + 3]);
case VarType::SInt:
return RowValuesToString((int)v.columns, v.displayAsHex, v.value.s32v[row * v.columns + 0],
v.value.s32v[row * v.columns + 1], v.value.s32v[row * v.columns + 2],
v.value.s32v[row * v.columns + 3]);
case VarType::SShort:
return RowValuesToString((int)v.columns, v.displayAsHex, v.value.s16v[row * v.columns + 0],
v.value.s16v[row * v.columns + 1], v.value.s16v[row * v.columns + 2],
v.value.s16v[row * v.columns + 3]);
case VarType::SByte:
return RowValuesToString((int)v.columns, v.displayAsHex, v.value.s8v[row * v.columns + 0],
v.value.s8v[row * v.columns + 1], v.value.s8v[row * v.columns + 2],
v.value.s8v[row * v.columns + 3]);
case VarType::GPUPointer: return ToQStr(v.GetPointer());
case VarType::ConstantBlock:
case VarType::ReadOnlyResource:
case VarType::ReadWriteResource:
case VarType::Sampler:
case VarType::Unknown: break;
}
return lit("???");
}
QString VarString(const ShaderVariable &v)
qrenderdoc/Code/QRDUtils.h
+1
View File
@@ -228,6 +228,7 @@ struct Formatter
static void shutdown();
static QString Format(double f, bool hex = false);
static QString Format(rdhalf f, bool hex = false) { return Format((float)f, hex); }
static QString HumanFormat(uint64_t u);
static QString Format(uint64_t u, bool hex = false)
{
qrenderdoc/Code/pyrenderdoc/pyconversion.h
+19
View File
@@ -364,6 +364,25 @@ struct TypeConversion<double, false>
static PyObject *ConvertToPy(const double &in) { return PyFloat_FromDouble(in); }
};
template <>
struct TypeConversion<rdhalf, false>
{
static int ConvertFromPy(PyObject *in, rdhalf &out)
{
if(!PyFloat_Check(in))
return SWIG_TypeError;
out.set(float(PyFloat_AsDouble(in)));
if(PyErr_Occurred())
return SWIG_OverflowError;
return SWIG_OK;
}
static PyObject *ConvertToPy(const rdhalf &in) { return PyFloat_FromDouble((float)in); }
};
// partial specialisation for enums, we just convert as their underlying type,
// whatever integer size that happens to be
template <typename T>
qrenderdoc/Code/pyrenderdoc/renderdoc.i
+2
View File
@@ -100,6 +100,7 @@ VA_IGNORE_REST_OF_FILE
%ignore rdcfixedarray::operator[];
%ignore rdcliteral;
%ignore rdcpair;
%ignore rdhalf;
%ignore bytebuf;
// special handling for RENDERDOC_GetDefaultCaptureOptions to transform output parameter to a return value
@@ -331,6 +332,7 @@ TEMPLATE_FIXEDARRAY_INSTANTIATE(rdcfixedarray, ResourceId, 8)
TEMPLATE_FIXEDARRAY_INSTANTIATE(rdcfixedarray, bool, 4)
TEMPLATE_FIXEDARRAY_INSTANTIATE(rdcfixedarray, bool, 8)
TEMPLATE_FIXEDARRAY_INSTANTIATE(rdcfixedarray, float, 16)
TEMPLATE_FIXEDARRAY_INSTANTIATE(rdcfixedarray, rdhalf, 16)
TEMPLATE_FIXEDARRAY_INSTANTIATE(rdcfixedarray, int32_t, 16)
TEMPLATE_FIXEDARRAY_INSTANTIATE(rdcfixedarray, uint32_t, 16)
TEMPLATE_FIXEDARRAY_INSTANTIATE(rdcfixedarray, double, 16)
qrenderdoc/Windows/ShaderViewer.cpp
+109 -56
View File
@@ -2593,6 +2593,10 @@ bool ShaderViewer::getVar(RDTreeWidgetItem *item, ShaderVariable *var, QString *
const QString xyzw = lit("xyzw");
size_t dataSize = VarTypeByteSize(ret.type);
if(dataSize == 0)
dataSize = 4;
for(uint32_t i = 0; i < mapping.variables.size(); i++)
{
const DebugVariableReference &r = mapping.variables[i];
@@ -2628,10 +2632,14 @@ bool ShaderViewer::getVar(RDTreeWidgetItem *item, ShaderVariable *var, QString *
}
}
if(mapping.type == VarType::Double || mapping.type == VarType::ULong)
if(dataSize == 8)
ret.value.u64v[i] = reg->value.u64v[r.component];
else
else if(dataSize == 4)
ret.value.u32v[i] = reg->value.u32v[r.component];
else if(dataSize == 2)
ret.value.u16v[i] = reg->value.u16v[r.component];
else
ret.value.u8v[i] = reg->value.u8v[r.component];
}
else
{
@@ -3393,7 +3401,7 @@ void ShaderViewer::updateWatchVariables()
"(\\[[0-9]+\\])?" // a literal-indexed array expression
"\\.?" // optional struct dot
")+)" // 1 or more chained identifiers
"(,[xfiudb])?" // optional typecast
"(,[iduxbof])?" // optional typecast
"$")); // end of the line
QRegularExpression identifierSliceRE(
@@ -3404,7 +3412,7 @@ void ShaderViewer::updateWatchVariables()
"(\\[[0-9]+\\])" // or a literal-indexed array expression
")")); // end capture
QRegularExpression swizzleRE(lit("^\\.[xyzwrgba]+$"));
QRegularExpression swizzleRE(lit("^\\.?[xyzwrgba]+$"));
QRegularExpressionMatch match = exprRE.match(expr);
@@ -3475,7 +3483,9 @@ void ShaderViewer::updateWatchVariables()
// member
if(swizzleRE.match(identifier).hasMatch() && identifiers.isEmpty())
{
swizzle = identifier.mid(1);
swizzle = identifier;
if(swizzle[0] == QLatin1Char('.'))
swizzle = swizzle.mid(1);
break;
}
@@ -3500,21 +3510,40 @@ void ShaderViewer::updateWatchVariables()
if(swizzle.isEmpty())
swizzle = lit("xyzw").left((int)var.columns);
size_t dataSize = VarTypeByteSize(var.type);
if(regcast == QLatin1Char(' '))
{
if(var.type == VarType::Double)
regcast = QLatin1Char('d');
else if(var.type == VarType::Float || var.type == VarType::Half)
regcast = QLatin1Char('f');
else if(var.type == VarType::ULong || var.type == VarType::UInt ||
var.type == VarType::UShort || var.type == VarType::UByte)
regcast = QLatin1Char('u');
else if(var.type == VarType::SLong || var.type == VarType::SInt ||
var.type == VarType::SShort || var.type == VarType::SByte ||
var.type == VarType::Bool)
regcast = QLatin1Char('i');
else if(var.type == VarType::Unknown)
regcast = ui->intView->isChecked() ? QLatin1Char('i') : QLatin1Char('f');
switch(var.type)
{
case VarType::Float:
case VarType::Double:
case VarType::Half: regcast = QLatin1Char('f'); break;
case VarType::Bool:
case VarType::ULong:
case VarType::UInt:
case VarType::UShort:
case VarType::UByte: regcast = QLatin1Char('u'); break;
case VarType::SLong:
case VarType::SInt:
case VarType::SShort:
case VarType::SByte: regcast = QLatin1Char('i'); break;
case VarType::GPUPointer:
regcast = QLatin1Char('#');
dataSize = 8;
break;
case VarType::ConstantBlock:
case VarType::ReadOnlyResource:
case VarType::ReadWriteResource:
case VarType::Sampler:
regcast = QLatin1Char('#');
dataSize = 4;
break;
case VarType::Unknown:
regcast = ui->intView->isChecked() ? QLatin1Char('i') : QLatin1Char('f');
dataSize = 4;
break;
}
}
QString val;
@@ -3533,29 +3562,58 @@ void ShaderViewer::updateWatchVariables()
if(swiz == QLatin1Char('w') || swiz == QLatin1Char('a'))
elindex = 3;
if(regcast == QLatin1Char('i'))
ShaderValue value = {};
memcpy(&value, var.value.u8v.data() + elindex * dataSize, dataSize);
if(regcast == QLatin1Char('#'))
{
val += Formatter::Format(var.value.s32v[elindex]);
if(var.type == VarType::GPUPointer)
val += ToQStr(var.GetPointer());
else
val += stringRep(var, 0);
}
else if(regcast == QLatin1Char('i') || regcast == QLatin1Char('d'))
{
if(dataSize == 8)
val += Formatter::Format(value.s64v[0]);
else if(dataSize == 4)
val += Formatter::Format(value.s32v[0]);
else if(dataSize == 2)
val += Formatter::Format(value.s16v[0]);
else
val += Formatter::Format(value.s8v[0]);
}
else if(regcast == QLatin1Char('f'))
{
val += Formatter::Format(var.value.f32v[elindex]);
if(dataSize == 8)
val += Formatter::Format(value.f64v[0]);
else if(dataSize == 4)
val += Formatter::Format(value.f32v[0]);
else
val += Formatter::Format(value.f16v[0]);
}
else if(regcast == QLatin1Char('u'))
{
val += Formatter::Format(var.value.u32v[elindex]);
val += Formatter::Format(value.u64v[0]);
}
else if(regcast == QLatin1Char('x'))
{
val += Formatter::Format(var.value.u32v[elindex], true);
if(dataSize == 8)
val += Formatter::Format(value.u64v[0], true);
else if(dataSize == 4)
val += Formatter::Format(value.u32v[0], true);
else if(dataSize == 2)
val += Formatter::Format(value.u16v[0], true);
else
val += Formatter::Format(value.u8v[0], true);
}
else if(regcast == QLatin1Char('b'))
{
val += QFormatStr("%1").arg(var.value.u32v[elindex], 32, 2, QLatin1Char('0'));
val += QFormatStr("%1").arg(value.u64v[0], (int)dataSize * 8, 2, QLatin1Char('0'));
}
else if(regcast == QLatin1Char('d'))
else if(regcast == QLatin1Char('o'))
{
val += Formatter::Format(var.value.f64v[elindex]);
val += QFormatStr("0%1").arg(value.u64v[0], 0, 8, QLatin1Char('0'));
}
if(s < swizzle.count() - 1)
@@ -3610,16 +3668,7 @@ RDTreeWidgetItem *ShaderViewer::makeSourceVariableNode(const ShaderVariable &var
{
QString typeName;
if(var.type == VarType::UInt)
typeName = lit("uint");
else if(var.type == VarType::SInt)
typeName = lit("int");
else if(var.type == VarType::Float)
typeName = lit("float");
else if(var.type == VarType::Double)
typeName = lit("double");
else if(var.type == VarType::Bool)
typeName = lit("bool");
typeName = ToQStr(var.type);
QString rowTypeName = typeName;
@@ -3673,16 +3722,7 @@ RDTreeWidgetItem *ShaderViewer::makeSourceVariableNode(const SourceVariableMappi
QString regNames, typeName;
QString value;
if(l.type == VarType::UInt)
typeName = lit("uint");
else if(l.type == VarType::SInt)
typeName = lit("int");
else if(l.type == VarType::Float)
typeName = lit("float");
else if(l.type == VarType::Double)
typeName = lit("double");
else if(l.type == VarType::Bool)
typeName = lit("bool");
typeName = ToQStr(l.type);
QList<RDTreeWidgetItem *> children;
@@ -3852,16 +3892,29 @@ RDTreeWidgetItem *ShaderViewer::makeSourceVariableNode(const SourceVariableMappi
regNames += QFormatStr("%1.%2").arg(r.name).arg(xyzw[r.component % 4]);
}
if(l.type == VarType::UInt)
value += Formatter::Format(reg->value.u32v[r.component]);
else if(l.type == VarType::SInt)
value += Formatter::Format(reg->value.s32v[r.component]);
else if(l.type == VarType::Bool)
value += Formatter::Format(reg->value.u32v[r.component] ? true : false);
else if(l.type == VarType::Float)
value += Formatter::Format(reg->value.f32v[r.component]);
else if(l.type == VarType::Double)
value += Formatter::Format(reg->value.f64v[r.component]);
switch(l.type)
{
case VarType::Float: value += Formatter::Format(reg->value.f32v[r.component]); break;
case VarType::Double: value += Formatter::Format(reg->value.f64v[r.component]); break;
case VarType::Half: value += Formatter::Format(reg->value.f16v[r.component]); break;
case VarType::Bool: Formatter::Format(reg->value.u32v[r.component] ? true : false);
case VarType::ULong: value += Formatter::Format(reg->value.u64v[r.component]); break;
case VarType::UInt: value += Formatter::Format(reg->value.u32v[r.component]); break;
case VarType::UShort: value += Formatter::Format(reg->value.u16v[r.component]); break;
case VarType::UByte: value += Formatter::Format(reg->value.u8v[r.component]); break;
case VarType::SLong: value += Formatter::Format(reg->value.s64v[r.component]); break;
case VarType::SInt: value += Formatter::Format(reg->value.s32v[r.component]); break;
case VarType::SShort: value += Formatter::Format(reg->value.s16v[r.component]); break;
case VarType::SByte: value += Formatter::Format(reg->value.s8v[r.component]); break;
case VarType::GPUPointer: value += ToQStr(reg->GetPointer()); break;
case VarType::ConstantBlock:
case VarType::ReadOnlyResource:
case VarType::ReadWriteResource:
case VarType::Sampler: value += stringRep(*reg, 0); break;
case VarType::Unknown:
qCritical() << "Unexpected unknown variable" << (QString)l.name;
break;
}
}
else
{