Use custom sort on shader messages for location/workgroup

* The default sorting is purely text based so 1,0,0 is followed by 10,0,0 not
  2,0,0. We customise the sort so it does the right thing per-column.

qrenderdoc/Widgets/Extended/RDTreeWidget.cpp

@@ -311,45 +311,49 @@ void RDTreeWidgetItem::sort(int column, Qt::SortOrder order)
 {
   ICaptureContext *ctx = getCaptureContext(m_widget);
 
-  std::sort(m_children.begin(), m_children.end(),
-            [ctx, column, order](const RDTreeWidgetItem *a, const RDTreeWidgetItem *b) {
-              QVariant va = a->data(column, Qt::DisplayRole);
-              QVariant vb = b->data(column, Qt::DisplayRole);
+  std::stable_sort(m_children.begin(), m_children.end(),
+                   [ctx, column, order](const RDTreeWidgetItem *a, const RDTreeWidgetItem *b) {
 
-              QString sa, sb;
+                     if(a->treeWidget()->m_SortComparison)
+                       return a->treeWidget()->m_SortComparison(column, order, a, b);
 
-              if(ctx)
-              {
-                sa = RichResourceTextFormat(*ctx, va);
-                sb = RichResourceTextFormat(*ctx, vb);
-              }
-              else
-              {
-                sa = va.toString();
-                sb = vb.toString();
-              }
+                     QVariant va = a->data(column, Qt::DisplayRole);
+                     QVariant vb = b->data(column, Qt::DisplayRole);
 
-              bool da_ok = false, db_ok = false;
-              double da = sa.toDouble(&da_ok);
-              double db = sb.toDouble(&db_ok);
+                     QString sa, sb;
 
-              int comp;
+                     if(ctx)
+                     {
+                       sa = RichResourceTextFormat(*ctx, va);
+                       sb = RichResourceTextFormat(*ctx, vb);
+                     }
+                     else
+                     {
+                       sa = va.toString();
+                       sb = vb.toString();
+                     }
 
-              if(da_ok && db_ok)
-              {
-                if(order == Qt::AscendingOrder)
-                  return da < db;
-                return da > db;
-              }
-              else
-              {
-                comp = QString::compare(sa, sb, Qt::CaseInsensitive);
-              }
+                     bool da_ok = false, db_ok = false;
+                     double da = sa.toDouble(&da_ok);
+                     double db = sb.toDouble(&db_ok);
 
-              if(order == Qt::AscendingOrder)
-                return comp < 0;
-              return comp > 0;
-            });
+                     int comp;
+
+                     if(da_ok && db_ok)
+                     {
+                       if(order == Qt::AscendingOrder)
+                         return da < db;
+                       return da > db;
+                     }
+                     else
+                     {
+                       comp = QString::compare(sa, sb, Qt::CaseInsensitive);
+                     }
+
+                     if(order == Qt::AscendingOrder)
+                       return comp < 0;
+                     return comp > 0;
+                   });
 
   for(RDTreeWidgetItem *child : m_children)
     child->sort(column, order);

qrenderdoc/Widgets/Extended/RDTreeWidget.h

@@ -236,6 +236,10 @@ public:
 
   void copyItem(QPoint pos, RDTreeWidgetItem *item);
 
+  typedef std::function<bool(int, Qt::SortOrder, const RDTreeWidgetItem *, const RDTreeWidgetItem *)>
+      ComparisonFunction;
+
+  void setSortComparison(ComparisonFunction comparison) { m_SortComparison = comparison; }
   void setColumns(const QStringList &columns);
   const QStringList &getHeaders() const { return m_headers; }
   QString headerText(int column) const { return m_headers[column]; }
@@ -293,6 +297,8 @@ private:
 
   QStringList m_headers;
 
+  ComparisonFunction m_SortComparison;
+
   bool m_queueUpdates = false;
 
   RDTreeWidgetItem *m_queuedItem;

qrenderdoc/Windows/ShaderMessageViewer.cpp

@@ -288,7 +288,7 @@ ShaderMessageViewer::ShaderMessageViewer(ICaptureContext &ctx, ShaderStageMask s
     else
       return;
 
-    ShaderMessage msg = m_Messages[msgIdx];
+    const ShaderMessage &msg = m_Messages[msgIdx];
 
     const ShaderReflection *refl = m_Ctx.CurPipelineState().GetShaderReflection(msg.stage);
 
@@ -380,7 +380,7 @@ ShaderMessageViewer::ShaderMessageViewer(ICaptureContext &ctx, ShaderStageMask s
       else
         return;
 
-      ShaderMessage msg = m_Messages[msgIdx];
+      const ShaderMessage &msg = m_Messages[msgIdx];
 
       m_Ctx.SetEventID({}, m_EID, m_EID);
 
@@ -442,6 +442,71 @@ ShaderMessageViewer::ShaderMessageViewer(ICaptureContext &ctx, ShaderStageMask s
 
   OnEventChanged(m_Ctx.CurEvent());
 
+  ui->messages->setSortComparison(
+      [this](int col, Qt::SortOrder order, const RDTreeWidgetItem *a, const RDTreeWidgetItem *b) {
+        if(order == Qt::DescendingOrder)
+          std::swap(a, b);
+
+        const ShaderMessage &am = m_Messages[a->tag().toInt()];
+        const ShaderMessage &bm = m_Messages[b->tag().toInt()];
+
+        if(col == 3)
+        {
+          return am.message < bm.message;
+        }
+        else if(col == 2 || m_OrigShaders[5] == ResourceId())
+        {
+          // sort by location either if it's selected, or if it's not dispatch in which case we
+          // default to location sorting (don't try to sort by the button-only columns that have no
+          // data)
+
+          // sort by stage first
+          if(am.stage != bm.stage)
+            return am.stage < bm.stage;
+
+          if(am.stage == ShaderStage::Vertex)
+          {
+            const ShaderVertexMessageLocation &aloc = am.location.vertex;
+            const ShaderVertexMessageLocation &bloc = bm.location.vertex;
+
+            if(aloc.view != bloc.view)
+              return aloc.view < bloc.view;
+            if(aloc.instance != bloc.instance)
+              return aloc.instance < bloc.instance;
+            return aloc.vertexIndex < bloc.vertexIndex;
+          }
+          else if(am.stage == ShaderStage::Pixel)
+          {
+            const ShaderPixelMessageLocation &aloc = am.location.pixel;
+            const ShaderPixelMessageLocation &bloc = bm.location.pixel;
+
+            if(aloc.x != bloc.x)
+              return aloc.x < bloc.x;
+            if(aloc.y != bloc.y)
+              return aloc.y < bloc.y;
+            if(aloc.primitive != bloc.primitive)
+              return aloc.primitive < bloc.primitive;
+            return aloc.sample < bloc.sample;
+          }
+          else if(am.stage == ShaderStage::Compute)
+          {
+            // column 2 is the thread column for compute
+            return am.location.compute.thread < bm.location.compute.thread;
+          }
+          else
+          {
+            // can't sort these, pretend they're all equal
+            return false;
+          }
+        }
+        else if(col == 1)
+        {
+          return am.location.compute.workgroup < bm.location.compute.workgroup;
+        }
+
+        return false;
+      });
+
   ui->messages->sortByColumn(sortColumn, Qt::SortOrder::AscendingOrder);
 
   for(int i = 0; i < 4; i++)

renderdoc/api/replay/rdcarray.h

@@ -57,8 +57,8 @@ struct ItemHelper
   static bool lessthanRange(const T *a, const T *b, size_t count)
   {
     for(size_t i = 0; i < count; i++)
-      if(a[i] < b[i])
-        return true;
+      if(!(a[i] == b[i]))
+        return a[i] < b[i];
 
     return false;
   }

renderdoc/driver/vulkan/vk_shader_feedback.cpp

@@ -1849,8 +1849,6 @@ void VulkanReplay::FetchShaderFeedback(uint32_t eventId)
           msg.location.pixel.y = location[0] & 0xffff;
           msg.location.pixel.sample = location[1];
           msg.location.pixel.primitive = location[2];
-
-          RDCLOG("pixel %u, %u", msg.location.pixel.x, msg.location.pixel.y);
         }
 
         msg.message = StringFormat::FmtArgs(fmt.effective_format.c_str(), args);