Experimental fix to allow for JDWP connections with different ID sizes

renderdoc/android/jdwp.cpp

@@ -309,7 +309,7 @@ bool InjectLibraries(const std::string &deviceID, Network::Socket *sock)
 
   // look up onCreate in the most derived class - since we can't guarantee that the base
   // application.app.onCreate() will get called.
-  methodID onCreate = conn.GetMethod(thisClass.Object, "onCreate", "()V");
+  methodID onCreate = conn.GetMethod(thisClass.RefType, "onCreate", "()V");
 
   if(onCreate == 0)
   {

renderdoc/android/jdwp.h

@@ -211,13 +211,28 @@ enum class Error : uint16_t
 
 struct CommandData;
 
-// we abstract the objectID size away, and always treat it as a uint64 (if it's actually 4 bytes, we
-// just only read/write the lower 4 bytes).
-struct objectID
+// different IDs in JDWP can be different sizes, but we want to basically treat them all the same.
+// For that reason we have a templated struct (jdwpID) which has all the functions we need, we
+// instantiate it for each *unique* ID type, and then further typedef for other IDs that are the
+// same.
+
+// we abstract the actual size away, and always treat an ID as a uint64 (if it's actually 4 bytes,
+// we just only read/write the lower 4 bytes).
+enum class IDType
+{
+  field,
+  method,
+  object,
+  refType,
+  frame
+};
+
+template <IDType>
+struct jdwpID
 {
 public:
-  objectID() = default;
-  objectID(uint64_t v) { data.u64 = v; }
+  jdwpID() = default;
+  jdwpID(uint64_t v) { data.u64 = v; }
   operator uint64_t() const { return size == 4 ? data.u32 : data.u64; }
   static int32_t getSize() { return size; }
   static void setSize(int32_t s)
@@ -243,21 +258,24 @@ private:
   static int32_t size;
 };
 
-// all object types are the same and we don't care too much about type safety, so we just typedef
-// them.
+typedef jdwpID<IDType::object> objectID;
 typedef objectID threadID;
 typedef objectID threadGroupID;
 typedef objectID stringID;
 typedef objectID classLoaderID;
 typedef objectID classObjectID;
 typedef objectID arrayID;
-typedef objectID referenceTypeID;
-typedef objectID classID;
-typedef objectID interfaceID;
-typedef objectID arrayTypeID;
-typedef objectID methodID;
-typedef objectID fieldID;
-typedef objectID frameID;
+
+// docs are weird - the protocol says referenceTypeID size is "same as objectID", but it has a
+// separate ID size. To be safe, keep it separate.
+typedef jdwpID<IDType::refType> referenceTypeID;
+typedef referenceTypeID classID;
+typedef referenceTypeID interfaceID;
+typedef referenceTypeID arrayTypeID;
+
+typedef jdwpID<IDType::method> methodID;
+typedef jdwpID<IDType::field> fieldID;
+typedef jdwpID<IDType::frame> frameID;
 
 struct taggedObjectID
 {
@@ -275,6 +293,7 @@ struct value
     byte Byte;
     char Char;
     objectID Object;
+    referenceTypeID RefType;
     float Float;
     double Double;
     int32_t Int;
@@ -423,10 +442,6 @@ CommandData &CommandData::Read(std::string &str);
 template <>
 CommandData &CommandData::Write(const std::string &str);
 template <>
-CommandData &CommandData::Read(objectID &id);
-template <>
-CommandData &CommandData::Write(const objectID &id);
-template <>
 CommandData &CommandData::Read(taggedObjectID &id);
 template <>
 CommandData &CommandData::Write(const taggedObjectID &id);
@@ -439,6 +454,28 @@ CommandData &CommandData::Read(Location &loc);
 template <>
 CommandData &CommandData::Write(const Location &loc);
 
+// objectID variants
+template <>
+CommandData &CommandData::Read(objectID &id);
+template <>
+CommandData &CommandData::Write(const objectID &id);
+template <>
+CommandData &CommandData::Read(referenceTypeID &id);
+template <>
+CommandData &CommandData::Write(const referenceTypeID &id);
+template <>
+CommandData &CommandData::Read(methodID &id);
+template <>
+CommandData &CommandData::Write(const methodID &id);
+template <>
+CommandData &CommandData::Read(fieldID &id);
+template <>
+CommandData &CommandData::Write(const fieldID &id);
+template <>
+CommandData &CommandData::Read(frameID &id);
+template <>
+CommandData &CommandData::Write(const frameID &id);
+
 typedef std::function<bool(const Event &evData)> EventFilterFunction;
 
 // A JDWP connection, with high-level helper functions that implement the protocol underneath

renderdoc/android/jdwp_connection.cpp

@@ -123,25 +123,61 @@ void Connection::SetupIDSizes()
       .Read(frameIDSize)
       .Done();
 
-  // we only support sizes that are all the same
-  if(fieldIDSize != methodIDSize || fieldIDSize != objectIDSize ||
-     fieldIDSize != referenceTypeIDSize || fieldIDSize != frameIDSize)
+  if(objectIDSize != referenceTypeIDSize)
   {
-    RDCLOG("Field ID sizes (%d %d %d %d %d) are not the same. Unsupported!", fieldIDSize,
-           methodIDSize, objectIDSize, referenceTypeIDSize, frameIDSize);
-    error = true;
-    return;
+    RDCWARN("objectID (%d) is not the same size as referenceTypeID (%d). Could cause problems!",
+            objectIDSize, referenceTypeIDSize);
   }
 
-  // we also only support 4 or 8 bytes
+  // we only support 4 or 8 bytes
   if(fieldIDSize != 4 && fieldIDSize != 8)
   {
-    RDCLOG("Field ID size %d is unsupported!", fieldIDSize);
+    RDCLOG("fieldID size %d is unsupported!", fieldIDSize);
     error = true;
     return;
   }
 
-  objectID::setSize(fieldIDSize);
+  fieldID::setSize(fieldIDSize);
+
+  // we only support 4 or 8 bytes
+  if(methodIDSize != 4 && methodIDSize != 8)
+  {
+    RDCLOG("methodID size %d is unsupported!", methodIDSize);
+    error = true;
+    return;
+  }
+
+  methodID::setSize(methodIDSize);
+
+  // we only support 4 or 8 bytes
+  if(objectIDSize != 4 && objectIDSize != 8)
+  {
+    RDCLOG("objectID size %d is unsupported!", objectIDSize);
+    error = true;
+    return;
+  }
+
+  objectID::setSize(objectIDSize);
+
+  // we only support 4 or 8 bytes
+  if(referenceTypeIDSize != 4 && referenceTypeIDSize != 8)
+  {
+    RDCLOG("referenceTypeID size %d is unsupported!", referenceTypeIDSize);
+    error = true;
+    return;
+  }
+
+  referenceTypeID::setSize(referenceTypeIDSize);
+
+  // we only support 4 or 8 bytes
+  if(frameIDSize != 4 && frameIDSize != 8)
+  {
+    RDCLOG("frameID size %d is unsupported!", frameIDSize);
+    error = true;
+    return;
+  }
+
+  frameID::setSize(frameIDSize);
 }
 
 void Connection::Suspend()

renderdoc/android/jdwp_util.cpp

@@ -33,6 +33,10 @@ namespace JDWP
 {
 uint32_t Command::idalloc = 42;
 int32_t objectID::size = 8;
+int32_t referenceTypeID::size = 8;
+int32_t methodID::size = 8;
+int32_t fieldID::size = 8;
+int32_t frameID::size = 8;
 
 uint32_t Command::Send(StreamWriter &writer)
 {
@@ -129,23 +133,6 @@ CommandData &CommandData::Write(const std::string &str)
   return *this;
 }
 
-template <>
-CommandData &CommandData::Read(objectID &id)
-{
-  ReadBytes(&id, objectID::getSize());
-  id.EndianSwap();
-  return *this;
-}
-
-template <>
-CommandData &CommandData::Write(const objectID &id)
-{
-  objectID tmp = id;
-  tmp.EndianSwap();
-  WriteBytes(&tmp, objectID::getSize());
-  return *this;
-}
-
 template <>
 CommandData &CommandData::Read(taggedObjectID &id)
 {
@@ -235,4 +222,90 @@ CommandData &CommandData::Write(const Location &loc)
   Write(loc.index);
   return *this;
 }
+
+// objectID variants
+template <>
+CommandData &CommandData::Read(objectID &id)
+{
+  ReadBytes(&id, objectID::getSize());
+  id.EndianSwap();
+  return *this;
+}
+
+template <>
+CommandData &CommandData::Write(const objectID &id)
+{
+  objectID tmp = id;
+  tmp.EndianSwap();
+  WriteBytes(&tmp, objectID::getSize());
+  return *this;
+}
+
+template <>
+CommandData &CommandData::Read(referenceTypeID &id)
+{
+  ReadBytes(&id, referenceTypeID::getSize());
+  id.EndianSwap();
+  return *this;
+}
+
+template <>
+CommandData &CommandData::Write(const referenceTypeID &id)
+{
+  referenceTypeID tmp = id;
+  tmp.EndianSwap();
+  WriteBytes(&tmp, referenceTypeID::getSize());
+  return *this;
+}
+
+template <>
+CommandData &CommandData::Read(methodID &id)
+{
+  ReadBytes(&id, methodID::getSize());
+  id.EndianSwap();
+  return *this;
+}
+
+template <>
+CommandData &CommandData::Write(const methodID &id)
+{
+  methodID tmp = id;
+  tmp.EndianSwap();
+  WriteBytes(&tmp, methodID::getSize());
+  return *this;
+}
+
+template <>
+CommandData &CommandData::Read(fieldID &id)
+{
+  ReadBytes(&id, fieldID::getSize());
+  id.EndianSwap();
+  return *this;
+}
+
+template <>
+CommandData &CommandData::Write(const fieldID &id)
+{
+  fieldID tmp = id;
+  tmp.EndianSwap();
+  WriteBytes(&tmp, fieldID::getSize());
+  return *this;
+}
+
+template <>
+CommandData &CommandData::Read(frameID &id)
+{
+  ReadBytes(&id, frameID::getSize());
+  id.EndianSwap();
+  return *this;
+}
+
+template <>
+CommandData &CommandData::Write(const frameID &id)
+{
+  frameID tmp = id;
+  tmp.EndianSwap();
+  WriteBytes(&tmp, frameID::getSize());
+  return *this;
+}
 };