Basic Metal capture support

Captures can be manually triggered from renderdoccmd capture <application> using F12 or from the UI on the in-development Metal replay branch.
The captures can be loaded and replayed on the in-development Metal replay branch.

The command buffer tracking and serialization are done by GPU submission order which is not necessarily the same as CPU commit order. The command buffer tracking for GPU submission order is currently using an rdcarray, this might change in the future to use a linked list if the performance of appending and deleting from the rdcarray becomes a performance bottleneck.

Does not include support for the presented MTLDrawable ie.
* Thumbnail generation of the final presented image.
* Serializing the presented texture ID.

Does not include support for initial state data.
No MTLBuffer data contents are serialized.

There is a lot missing and a lot of TODOs. 
This is the basic structure for capturing which is then built upon. 

Includes:
* register the Metal device as a frame capturer ie. AddDeviceFrameCapturer
* logic for triggering captures at Present ie. AddActiveDriver, StartFrameCapture, EndFrameCapture.
* Stopped declaring MetalResourceManager as a friend of WrappedMTLDevice which meant making the initial state-related APIs public instead of private.
* IFrameCapturer interface APIs
* Command buffer tracking for including in the output capture CaptureCmdBufCommit and CaptureCmdBufEnqueue.
* Serialise_MTLCreateSystemDefaultDevice(SerialiserType &ser)
* A helper class MetalCapturer which is derived from IFrameCapturer and registered with the RenderDoc instance. This is because Wrapped Metal classes can't have virtual tables as a requirement for how the C++ and Objective C overlay is implemented.
* The frame capture chunk and API AddFrameCaptureRecordChunk
* MetalInitParams data and serialization.

Helper Methods and Members in MTLDevice
* WaitForGPU()
* MTL::CommandQueue *m_mtlCommandQueue which is used to implement WaitForGPU()
* CaptureClearSubmittedCmdBuffers() & CaptureCmdBufSubmit()
* RegisterMetalLayer() & UnregisterMetalLayer() used to track active Metal swapchains

Details on the memory lifetime for WrappedMTLCommandBuffer

retain the real resource in WrappedMTLCommandBuffer::commit()
release the real resource when no longer needed to be tracked: for background capture in during WrappedMTLDevice::CaptureCmdBufSubmit, for active capture in WrappedMTLDevice::EndFrameCapture.

During capture (Background or Active)
* AddRef() record when command buffer is enqueued (explicit or implicit)
* Delete() record at end of command buffer submit

During Active capture
* AddRef() record in command buffer submit
* Delete() submitted command buffers as part of finalizing the capture

Added TrackedCAMetalLayer & ObjCTrackedCAMetalLayer to track the lifetime of CA::MetalLayer.
The CA::MetalLayer is tracked in the hook for CAMetalLayer::nextDrawable(), with ObjCTrackedCAMetalLayer set as an association to the CAMetalLayer. Then ObjCTrackedCAMetalLayer::dealloc() triggers ending the tracking.

renderdoc/core/core.cpp

@@ -195,6 +195,7 @@ rdcstr DoStringise(const RDCDriver &el)
     STRINGISE_ENUM_CLASS(D3D8);
     STRINGISE_ENUM_CLASS(Image);
     STRINGISE_ENUM_CLASS(Vulkan);
+    STRINGISE_ENUM_CLASS(Metal);
   }
   END_ENUM_STRINGISE();
 }

renderdoc/core/core.h

@@ -219,6 +219,7 @@ enum class RDCDriver : uint32_t
   Vulkan = 8,
   OpenGLES = 9,
   D3D8 = 10,
+  Metal = 11,
   MaxBuiltin,
   Custom = 100000,
   Custom0 = Custom,

renderdoc/driver/metal/metal_buffer.cpp

@@ -100,8 +100,7 @@ void WrappedMTLBuffer::didModifyRange(NS::Range &range)
         Serialise_didModifyRange(ser, range);
         chunk = scope.Get();
       }
-      // TODO: add to the frame capture record chunk
-      // m_Device->AddFrameCaptureRecordChunk(chunk);
+      m_Device->AddFrameCaptureRecordChunk(chunk);
     }
   }
   else

renderdoc/driver/metal/metal_command_buffer.cpp

@@ -179,7 +179,7 @@ void WrappedMTLCommandBuffer::presentDrawable(MTL::Drawable *drawable)
     }
     MetalResourceRecord *bufferRecord = GetRecord(this);
     bufferRecord->AddChunk(chunk);
-    bufferRecord->cmdInfo->flags |= MetalCmdBufferStatus::Presented;
+    bufferRecord->cmdInfo->presented = true;
     bufferRecord->cmdInfo->outputLayer = ObjC::Get_Layer(drawable);
     bufferRecord->cmdInfo->backBuffer = GetWrapped(mtlBackBuffer);
   }
@@ -206,26 +206,17 @@ bool WrappedMTLCommandBuffer::Serialise_commit(SerialiserType &ser)
 
 void WrappedMTLCommandBuffer::commit()
 {
-  SERIALISE_TIME_CALL(Unwrap(this)->commit());
-  if(IsCaptureMode(m_State))
+  MTL::CommandBuffer *mtlCommandBuffer = Unwrap(this);
+  bool isCapture = IsCaptureMode(m_State);
+  // During capture keep the real resource alive
+  // It will be released when it is no longer required to be tracked
+  if(isCapture)
+    mtlCommandBuffer->retain();
+  SERIALISE_TIME_CALL(mtlCommandBuffer->commit());
+  if(isCapture)
   {
-    Chunk *chunk = NULL;
-    {
-      CACHE_THREAD_SERIALISER();
-      SCOPED_SERIALISE_CHUNK(MetalChunk::MTLCommandBuffer_commit);
-      Serialise_commit(ser);
-      chunk = scope.Get();
-    }
     MetalResourceRecord *bufferRecord = GetRecord(this);
-    bufferRecord->AddChunk(chunk);
-
-    if(IsActiveCapturing(m_State))
-    {
-      bufferRecord->AddRef();
-      bufferRecord->MarkResourceFrameReferenced(GetResID(m_CommandQueue), eFrameRef_Read);
-      // pull in frame refs from this command buffer
-      bufferRecord->AddResourceReferences(GetResourceManager());
-    }
+    m_Device->CaptureCmdBufCommit(bufferRecord);
   }
   else
   {
@@ -261,6 +252,7 @@ void WrappedMTLCommandBuffer::enqueue()
     }
     MetalResourceRecord *bufferRecord = GetRecord(this);
     bufferRecord->AddChunk(chunk);
+    m_Device->CaptureCmdBufEnqueue(bufferRecord);
   }
   else
   {

renderdoc/driver/metal/metal_core.cpp

@@ -23,6 +23,8 @@
  ******************************************************************************/
 
 #include "metal_core.h"
+#include "serialise/rdcfile.h"
+#include "metal_command_buffer.h"
 #include "metal_device.h"
 
 WriteSerialiser &WrappedMTLDevice::GetThreadSerialiser()
@@ -53,3 +55,464 @@ WriteSerialiser &WrappedMTLDevice::GetThreadSerialiser()
 
   return *ser;
 }
+
+void WrappedMTLDevice::WaitForGPU()
+{
+  MTL::CommandBuffer *mtlCommandBuffer = m_mtlCommandQueue->commandBuffer();
+  mtlCommandBuffer->commit();
+  mtlCommandBuffer->waitUntilCompleted();
+}
+
+template <typename SerialiserType>
+bool WrappedMTLDevice::Serialise_BeginCaptureFrame(SerialiserType &ser)
+{
+  // TODO: serialise image references and states
+
+  SERIALISE_CHECK_READ_ERRORS();
+
+  return true;
+}
+
+void WrappedMTLDevice::StartFrameCapture(DeviceOwnedWindow devWnd)
+{
+  if(!IsBackgroundCapturing(m_State))
+    return;
+
+  RDCLOG("Starting capture");
+  {
+    SCOPED_LOCK(m_CaptureCommandBuffersLock);
+    RDCASSERT(m_CaptureCommandBuffersSubmitted.empty());
+  }
+
+  m_CaptureTimer.Restart();
+
+  GetResourceManager()->ResetCaptureStartTime();
+
+  m_AppControlledCapture = true;
+
+  FrameDescription frame;
+  frame.frameNumber = ~0U;
+  frame.captureTime = Timing::GetUnixTimestamp();
+  m_CapturedFrames.push_back(frame);
+
+  GetResourceManager()->ClearReferencedResources();
+  // TODO: handle tracked memory
+
+  // need to do all this atomically so that no other commands
+  // will check to see if they need to mark dirty or
+  // mark pending dirty and go into the frame record.
+  {
+    SCOPED_WRITELOCK(m_CapTransitionLock);
+
+    GetResourceManager()->PrepareInitialContents();
+
+    RDCDEBUG("Attempting capture");
+    m_FrameCaptureRecord->DeleteChunks();
+    m_State = CaptureState::ActiveCapturing;
+  }
+
+  GetResourceManager()->MarkResourceFrameReferenced(GetResID(this), eFrameRef_Read);
+
+  // TODO: are there other resources that need to be marked as frame referenced
+}
+
+void WrappedMTLDevice::EndCaptureFrame()
+{
+  CACHE_THREAD_SERIALISER();
+  ser.SetActionChunk();
+  SCOPED_SERIALISE_CHUNK(SystemChunk::CaptureEnd);
+
+  // TODO: serialise the presented image
+
+  m_FrameCaptureRecord->AddChunk(scope.Get());
+}
+
+bool WrappedMTLDevice::EndFrameCapture(DeviceOwnedWindow devWnd)
+{
+  if(!IsActiveCapturing(m_State))
+    return true;
+
+  // TODO: find the window and drawable being captured
+
+  RDCLOG("Finished capture, Frame %u", m_CapturedFrames.back().frameNumber);
+
+  // TODO: mark the drawable and its images as frame referenced
+
+  // atomically transition to IDLE
+  {
+    SCOPED_WRITELOCK(m_CapTransitionLock);
+    EndCaptureFrame();
+    m_State = CaptureState::BackgroundCapturing;
+  }
+
+  {
+    SCOPED_LOCK(m_CaptureCommandBuffersLock);
+    // wait for the GPU to be idle
+    for(MetalResourceRecord *record : m_CaptureCommandBuffersSubmitted)
+    {
+      WrappedMTLCommandBuffer *commandBuffer = (WrappedMTLCommandBuffer *)(record->m_Resource);
+      Unwrap(commandBuffer)->waitUntilCompleted();
+      // Remove the reference on the real resource added during commit()
+      Unwrap(commandBuffer)->release();
+    }
+
+    if(m_CaptureCommandBuffersSubmitted.empty())
+      WaitForGPU();
+  }
+
+  // TODO: get the backbuffer to generate the thumbnail image
+  RenderDoc::FramePixels fp;
+
+  RDCFile *rdc =
+      RenderDoc::Inst().CreateRDC(RDCDriver::Metal, m_CapturedFrames.back().frameNumber, fp);
+
+  StreamWriter *captureWriter = NULL;
+
+  if(rdc)
+  {
+    SectionProperties props;
+
+    // Compress with LZ4 so that it's fast
+    props.flags = SectionFlags::LZ4Compressed;
+    props.version = m_SectionVersion;
+    props.type = SectionType::FrameCapture;
+
+    captureWriter = rdc->WriteSection(props);
+  }
+  else
+  {
+    captureWriter = new StreamWriter(StreamWriter::InvalidStream);
+  }
+
+  uint64_t captureSectionSize = 0;
+
+  {
+    WriteSerialiser ser(captureWriter, Ownership::Stream);
+
+    ser.SetChunkMetadataRecording(GetThreadSerialiser().GetChunkMetadataRecording());
+    ser.SetUserData(GetResourceManager());
+
+    {
+      m_InitParams.Set(Unwrap(this), m_ID);
+      SCOPED_SERIALISE_CHUNK(SystemChunk::DriverInit, m_InitParams.GetSerialiseSize());
+      SERIALISE_ELEMENT(m_InitParams);
+    }
+
+    RDCDEBUG("Inserting Resource Serialisers");
+    GetResourceManager()->InsertReferencedChunks(ser);
+    GetResourceManager()->InsertInitialContentsChunks(ser);
+
+    RDCDEBUG("Creating Capture Scope");
+    GetResourceManager()->Serialise_InitialContentsNeeded(ser);
+    // TODO: memory references
+
+    // need over estimate of chunk size when writing directly to file
+    {
+      SCOPED_SERIALISE_CHUNK(SystemChunk::CaptureScope, 16);
+      Serialise_CaptureScope(ser);
+    }
+
+    {
+      uint64_t maxCaptureBeginChunkSizeInBytes = 16;
+      SCOPED_SERIALISE_CHUNK(SystemChunk::CaptureBegin, maxCaptureBeginChunkSizeInBytes);
+      Serialise_BeginCaptureFrame(ser);
+    }
+
+    // don't need to lock access to m_CaptureCommandBuffersSubmitted as
+    // no longer in active capture (the transition is thread-protected)
+    // nothing will be pushed to the vector
+
+    {
+      std::map<int64_t, Chunk *> recordlist;
+      size_t countCmdBuffers = m_CaptureCommandBuffersSubmitted.size();
+      // ensure all command buffer records within the frame even if recorded before
+      // serialised order must be preserved
+      for(MetalResourceRecord *record : m_CaptureCommandBuffersSubmitted)
+      {
+        size_t prevSize = recordlist.size();
+        (void)prevSize;
+        record->Insert(recordlist);
+      }
+
+      size_t prevSize = recordlist.size();
+      (void)prevSize;
+      m_FrameCaptureRecord->Insert(recordlist);
+      RDCDEBUG("Adding %zu/%zu frame capture chunks to file serialiser",
+               recordlist.size() - prevSize, recordlist.size());
+
+      float num = float(recordlist.size());
+      float idx = 0.0f;
+
+      for(auto it = recordlist.begin(); it != recordlist.end(); ++it)
+      {
+        RenderDoc::Inst().SetProgress(CaptureProgress::SerialiseFrameContents, idx / num);
+        idx += 1.0f;
+        it->second->Write(ser);
+      }
+    }
+    captureSectionSize = captureWriter->GetOffset();
+  }
+
+  RDCLOG("Captured Metal frame with %f MB capture section in %f seconds",
+         double(captureSectionSize) / (1024.0 * 1024.0), m_CaptureTimer.GetMilliseconds() / 1000.0);
+
+  RenderDoc::Inst().FinishCaptureWriting(rdc, m_CapturedFrames.back().frameNumber);
+
+  // delete tracked cmd buffers - had to keep them alive until after serialiser flush.
+  CaptureClearSubmittedCmdBuffers();
+
+  GetResourceManager()->ResetLastWriteTimes();
+  GetResourceManager()->MarkUnwrittenResources();
+
+  // TODO: handle memory resources in the resource manager
+
+  GetResourceManager()->ClearReferencedResources();
+  GetResourceManager()->FreeInitialContents();
+
+  // TODO: handle memory resources in the initial contents
+
+  return true;
+}
+
+bool WrappedMTLDevice::DiscardFrameCapture(DeviceOwnedWindow devWnd)
+{
+  if(!IsActiveCapturing(m_State))
+    return true;
+
+  RDCLOG("Discarding frame capture.");
+
+  RenderDoc::Inst().FinishCaptureWriting(NULL, m_CapturedFrames.back().frameNumber);
+
+  m_CapturedFrames.pop_back();
+
+  // atomically transition to IDLE
+  {
+    SCOPED_WRITELOCK(m_CapTransitionLock);
+    m_State = CaptureState::BackgroundCapturing;
+  }
+
+  CaptureClearSubmittedCmdBuffers();
+
+  GetResourceManager()->MarkUnwrittenResources();
+
+  // TODO: handle memory resources in the resource manager
+
+  GetResourceManager()->ClearReferencedResources();
+  GetResourceManager()->FreeInitialContents();
+
+  // TODO: handle memory resources in the initial contents
+
+  return true;
+}
+
+template <typename SerialiserType>
+bool WrappedMTLDevice::Serialise_CaptureScope(SerialiserType &ser)
+{
+  SERIALISE_ELEMENT_LOCAL(frameNumber, m_CapturedFrames.back().frameNumber);
+
+  SERIALISE_CHECK_READ_ERRORS();
+
+  if(IsReplayingAndReading())
+  {
+    // TODO: implement RD MTL replay
+  }
+  return true;
+}
+
+void WrappedMTLDevice::CaptureCmdBufSubmit(MetalResourceRecord *record)
+{
+  RDCASSERTEQUAL(record->cmdInfo->status, MetalCmdBufferStatus::Submitted);
+  RDCASSERT(IsCaptureMode(m_State));
+  WrappedMTLCommandBuffer *commandBuffer = (WrappedMTLCommandBuffer *)(record->m_Resource);
+  if(IsActiveCapturing(m_State))
+  {
+    Chunk *chunk = NULL;
+    std::unordered_set<ResourceId> refIDs;
+    // The record will get deleted at the end of active frame capture
+    record->AddRef();
+    record->AddReferencedIDs(refIDs);
+    // snapshot/detect any CPU modifications to the contents
+    // of referenced MTLBuffer with shared storage mode
+    for(auto it = refIDs.begin(); it != refIDs.end(); ++it)
+    {
+      ResourceId id = *it;
+      MetalResourceRecord *refRecord = GetResourceManager()->GetResourceRecord(id);
+      if(refRecord->m_Type == eResBuffer)
+      {
+        // TODO: capture CPU modified buffers
+      }
+    }
+    record->MarkResourceFrameReferenced(GetResID(commandBuffer->GetCommandQueue()), eFrameRef_Read);
+    // pull in frame refs from this command buffer
+    record->AddResourceReferences(GetResourceManager());
+    {
+      CACHE_THREAD_SERIALISER();
+      SCOPED_SERIALISE_CHUNK(MetalChunk::MTLCommandBuffer_commit);
+      commandBuffer->Serialise_commit(ser);
+      chunk = scope.Get();
+    }
+    record->AddChunk(chunk);
+    m_CaptureCommandBuffersSubmitted.push_back(record);
+  }
+  else
+  {
+    // Remove the reference on the real resource added during commit()
+    Unwrap(commandBuffer)->release();
+  }
+  if(record->cmdInfo->presented)
+  {
+    AdvanceFrame();
+    Present(record);
+  }
+  // In background or active capture mode the record reference is incremented in
+  // CaptureCmdBufEnqueue
+  record->Delete(GetResourceManager());
+}
+
+void WrappedMTLDevice::CaptureCmdBufCommit(MetalResourceRecord *record)
+{
+  SCOPED_LOCK(m_CaptureCommandBuffersLock);
+  if(record->cmdInfo->status != MetalCmdBufferStatus::Enqueued)
+    CaptureCmdBufEnqueue(record);
+
+  RDCASSERTEQUAL(record->cmdInfo->status, MetalCmdBufferStatus::Enqueued);
+  record->cmdInfo->status = MetalCmdBufferStatus::Committed;
+
+  size_t countSubmitted = 0;
+  for(MetalResourceRecord *record : m_CaptureCommandBuffersEnqueued)
+  {
+    if(record->cmdInfo->status == MetalCmdBufferStatus::Committed)
+    {
+      record->cmdInfo->status = MetalCmdBufferStatus::Submitted;
+      ++countSubmitted;
+      CaptureCmdBufSubmit(record);
+      continue;
+    }
+    break;
+  };
+  m_CaptureCommandBuffersEnqueued.erase(0, countSubmitted);
+}
+
+void WrappedMTLDevice::CaptureCmdBufEnqueue(MetalResourceRecord *record)
+{
+  SCOPED_LOCK(m_CaptureCommandBuffersLock);
+  RDCASSERTEQUAL(record->cmdInfo->status, MetalCmdBufferStatus::Unknown);
+  record->cmdInfo->status = MetalCmdBufferStatus::Enqueued;
+  record->AddRef();
+  m_CaptureCommandBuffersEnqueued.push_back(record);
+
+  RDCDEBUG("Enqueing CommandBufferRecord %s %d", ToStr(record->GetResourceID()).c_str(),
+           m_CaptureCommandBuffersEnqueued.count());
+}
+
+void WrappedMTLDevice::AdvanceFrame()
+{
+  if(IsBackgroundCapturing(m_State))
+    RenderDoc::Inst().Tick();
+
+  m_FrameCounter++;    // first present becomes frame #1, this function is at the end of the frame
+}
+
+void WrappedMTLDevice::FirstFrame()
+{
+  // if we have to capture the first frame, begin capturing immediately
+  if(IsBackgroundCapturing(m_State) && RenderDoc::Inst().ShouldTriggerCapture(0))
+  {
+    RenderDoc::Inst().StartFrameCapture(DeviceOwnedWindow(this, NULL));
+
+    m_AppControlledCapture = false;
+    m_CapturedFrames.back().frameNumber = 0;
+  }
+}
+
+void WrappedMTLDevice::Present(MetalResourceRecord *record)
+{
+  WrappedMTLTexture *backBuffer = record->cmdInfo->backBuffer;
+  {
+    SCOPED_LOCK(m_CapturePotentialBackBuffersLock);
+    if(m_CapturePotentialBackBuffers.count(backBuffer) == 0)
+    {
+      RDCERR("Capture ignoring Present called on unknown backbuffer");
+      return;
+    }
+  }
+
+  CA::MetalLayer *outputLayer = record->cmdInfo->outputLayer;
+  DeviceOwnedWindow devWnd(this, outputLayer);
+
+  bool activeWindow = RenderDoc::Inst().IsActiveWindow(devWnd);
+
+  RenderDoc::Inst().AddActiveDriver(RDCDriver::Metal, true);
+
+  if(!activeWindow)
+    return;
+
+  if(IsActiveCapturing(m_State) && !m_AppControlledCapture)
+    RenderDoc::Inst().EndFrameCapture(devWnd);
+
+  if(RenderDoc::Inst().ShouldTriggerCapture(m_FrameCounter) && IsBackgroundCapturing(m_State))
+  {
+    RenderDoc::Inst().StartFrameCapture(devWnd);
+
+    m_AppControlledCapture = false;
+    m_CapturedFrames.back().frameNumber = m_FrameCounter;
+  }
+}
+
+void WrappedMTLDevice::CaptureClearSubmittedCmdBuffers()
+{
+  SCOPED_LOCK(m_CaptureCommandBuffersLock);
+  for(MetalResourceRecord *record : m_CaptureCommandBuffersSubmitted)
+  {
+    record->Delete(GetResourceManager());
+  }
+
+  m_CaptureCommandBuffersSubmitted.clear();
+}
+
+void WrappedMTLDevice::RegisterMetalLayer(CA::MetalLayer *mtlLayer)
+{
+  SCOPED_LOCK(m_CaptureOutputLayersLock);
+  if(m_CaptureOutputLayers.count(mtlLayer) == 0)
+  {
+    m_CaptureOutputLayers.insert(mtlLayer);
+    TrackedCAMetalLayer::Track(mtlLayer, this);
+
+    DeviceOwnedWindow devWnd(this, mtlLayer);
+    RenderDoc::Inst().AddFrameCapturer(devWnd, &m_Capturer);
+  }
+}
+
+void WrappedMTLDevice::UnregisterMetalLayer(CA::MetalLayer *mtlLayer)
+{
+  SCOPED_LOCK(m_CaptureOutputLayersLock);
+  RDCASSERT(m_CaptureOutputLayers.count(mtlLayer));
+  m_CaptureOutputLayers.erase(mtlLayer);
+
+  DeviceOwnedWindow devWnd(this, mtlLayer);
+  RenderDoc::Inst().RemoveFrameCapturer(devWnd);
+}
+
+MetalInitParams::MetalInitParams()
+{
+  memset(this, 0, sizeof(MetalInitParams));
+}
+
+uint64_t MetalInitParams::GetSerialiseSize()
+{
+  size_t ret = sizeof(*this);
+  return (uint64_t)ret;
+}
+
+void MetalInitParams::Set(MTL::Device *pRealDevice, ResourceId device)
+{
+  DeviceID = device;
+}
+
+template <typename SerialiserType>
+void DoSerialise(SerialiserType &ser, MetalInitParams &el)
+{
+  SERIALISE_MEMBER(DeviceID).TypedAs("MTLDevice"_lit);
+}
+
+INSTANTIATE_SERIALISE_TYPE(MetalInitParams);

renderdoc/driver/metal/metal_core.h

@@ -28,8 +28,16 @@
 
 struct MetalInitParams
 {
+  MetalInitParams();
+  void Set(MTL::Device *pDevice, ResourceId device);
+
+  // update this when adding/removing members
+  uint64_t GetSerialiseSize();
+
   // check if a frame capture section version is supported
   static const uint64_t CurrentVersion = 0x1;
+
+  ResourceId DeviceID;
 };
 
 DECLARE_REFLECTION_STRUCT(MetalInitParams);

renderdoc/driver/metal/metal_device.cpp

@@ -33,7 +33,7 @@
 #include "metal_texture.h"
 
 WrappedMTLDevice::WrappedMTLDevice(MTL::Device *realMTLDevice, ResourceId objId)
-    : WrappedMTLObject(realMTLDevice, objId, this, GetStateRef())
+    : WrappedMTLObject(realMTLDevice, objId, this, GetStateRef()), m_Capturer(*this)
 {
   AllocateObjCBridge(this);
   m_Device = this;
@@ -46,21 +46,53 @@ WrappedMTLDevice::WrappedMTLDevice(MTL::Device *realMTLDevice, ResourceId objId)
     m_State = CaptureState::BackgroundCapturing;
   }
 
+  m_SectionVersion = MetalInitParams::CurrentVersion;
+
   threadSerialiserTLSSlot = Threading::AllocateTLSSlot();
 
   m_ResourceManager = new MetalResourceManager(m_State, this);
+
+  if(!RenderDoc::Inst().IsReplayApp())
+  {
+    m_FrameCaptureRecord = GetResourceManager()->AddResourceRecord(ResourceIDGen::GetNewUniqueID());
+    m_FrameCaptureRecord->DataInSerialiser = false;
+    m_FrameCaptureRecord->Length = 0;
+    m_FrameCaptureRecord->InternalResource = true;
+  }
+  else
+  {
+    m_FrameCaptureRecord = NULL;
+
+    ResourceIDGen::SetReplayResourceIDs();
+  }
+
   RDCASSERT(m_Device == this);
   GetResourceManager()->AddCurrentResource(objId, this);
-}
 
-WrappedMTLDevice *WrappedMTLDevice::MTLCreateSystemDefaultDevice(MTL::Device *realMTLDevice)
-{
-  MTLFixupForMetalDriverAssert();
-  MTLHookObjcMethods();
-  ResourceId objId = ResourceIDGen::GetNewUniqueID();
-  WrappedMTLDevice *wrappedMTLDevice = new WrappedMTLDevice(realMTLDevice, objId);
+  if(IsCaptureMode(m_State))
+  {
+    Chunk *chunk = NULL;
 
-  return wrappedMTLDevice;
+    {
+      CACHE_THREAD_SERIALISER();
+
+      SCOPED_SERIALISE_CHUNK(MetalChunk::MTLCreateSystemDefaultDevice);
+      Serialise_MTLCreateSystemDefaultDevice(ser);
+      chunk = scope.Get();
+    }
+
+    MetalResourceRecord *record = GetResourceManager()->AddResourceRecord(this);
+    record->AddChunk(chunk);
+  }
+  else
+  {
+    // TODO: implement RD MTL replay
+  }
+
+  RenderDoc::Inst().AddDeviceFrameCapturer(this, &m_Capturer);
+
+  m_mtlCommandQueue = Unwrap(this)->newCommandQueue();
+  FirstFrame();
 }
 
 IMP WrappedMTLDevice::g_real_CAMetalLayer_nextDrawable;
@@ -68,6 +100,11 @@ uint64_t WrappedMTLDevice::g_nextDrawableTLSSlot;
 
 MTL::Drawable *hooked_CAMetalLayer_nextDrawable(id self, SEL _cmd)
 {
+  CA::MetalLayer *mtlLayer = (CA::MetalLayer *)self;
+  MTL::Device *mtlDevice = mtlLayer->device();
+  RDCASSERT(object_getClass(mtlDevice) == objc_getClass("ObjCBridgeMTLDevice"));
+  GetWrapped(mtlDevice)->RegisterMetalLayer(mtlLayer);
+
   RDCASSERTEQUAL(Threading::GetTLSValue(WrappedMTLDevice::g_nextDrawableTLSSlot), 0);
   Threading::SetTLSValue(WrappedMTLDevice::g_nextDrawableTLSSlot, (void *)(uintptr_t) true);
   MTL::Drawable *drawable =
@@ -108,6 +145,30 @@ void WrappedMTLDevice::MTLFixupForMetalDriverAssert()
 
 // Serialised MTLDevice APIs
 
+template <typename SerialiserType>
+bool WrappedMTLDevice::Serialise_MTLCreateSystemDefaultDevice(SerialiserType &ser)
+{
+  SERIALISE_ELEMENT_LOCAL(Device, GetResID(this)).TypedAs("MTLDevice"_lit);
+
+  SERIALISE_CHECK_READ_ERRORS();
+
+  if(IsReplayingAndReading())
+  {
+    // TODO: implement RD MTL replay
+  }
+  return true;
+}
+
+WrappedMTLDevice *WrappedMTLDevice::MTLCreateSystemDefaultDevice(MTL::Device *realMTLDevice)
+{
+  MTLFixupForMetalDriverAssert();
+  MTLHookObjcMethods();
+  ResourceId objId = ResourceIDGen::GetNewUniqueID();
+  WrappedMTLDevice *wrappedMTLDevice = new WrappedMTLDevice(realMTLDevice, objId);
+
+  return wrappedMTLDevice;
+}
+
 template <typename SerialiserType>
 bool WrappedMTLDevice::Serialise_newCommandQueue(SerialiserType &ser, WrappedMTLCommandQueue *queue)
 {
@@ -539,8 +600,8 @@ WrappedMTLTexture *WrappedMTLDevice::Common_NewTexture(RDMTL::TextureDescriptor
     if(IsCaptureMode(m_State))
     {
       {
-        SCOPED_LOCK(m_PotentialBackBuffersLock);
-        m_PotentialBackBuffers.insert(wrappedMTLTexture);
+        SCOPED_LOCK(m_CapturePotentialBackBuffersLock);
+        m_CapturePotentialBackBuffers.insert(wrappedMTLTexture);
       }
     }
   }
@@ -578,6 +639,7 @@ WrappedMTLBuffer *WrappedMTLDevice::Common_NewBuffer(bool withBytes, const void
   return wrappedMTLBuffer;
 }
 
+INSTANTIATE_FUNCTION_SERIALISED(WrappedMTLDevice, bool, MTLCreateSystemDefaultDevice);
 INSTANTIATE_FUNCTION_WITH_RETURN_SERIALISED(WrappedMTLDevice, WrappedMTLCommandQueue *,
                                             newCommandQueue);
 INSTANTIATE_FUNCTION_WITH_RETURN_SERIALISED(WrappedMTLDevice, WrappedMTLLibrary *, newDefaultLibrary);

renderdoc/driver/metal/metal_device.h

@@ -25,15 +25,33 @@
 #pragma once
 
 #include "metal_common.h"
+#include "metal_core.h"
 #include "metal_manager.h"
 
+class WrappedMTLDevice;
+
+class MetalCapturer : public IFrameCapturer
+{
+public:
+  MetalCapturer(WrappedMTLDevice &device) : m_Device(device) {}
+  // IFrameCapturer interface
+  RDCDriver GetFrameCaptureDriver() { return RDCDriver::Metal; }
+  void StartFrameCapture(DeviceOwnedWindow devWnd);
+  bool EndFrameCapture(DeviceOwnedWindow devWnd);
+  bool DiscardFrameCapture(DeviceOwnedWindow devWnd);
+  // IFrameCapturer interface
+
+private:
+  WrappedMTLDevice &m_Device;
+};
+
 class WrappedMTLDevice : public WrappedMTLObject
 {
-  friend class MetalResourceManager;
-
 public:
   WrappedMTLDevice(MTL::Device *realMTLDevice, ResourceId objId);
   ~WrappedMTLDevice() {}
+  template <typename SerialiserType>
+  bool Serialise_MTLCreateSystemDefaultDevice(SerialiserType &ser);
   static WrappedMTLDevice *MTLCreateSystemDefaultDevice(MTL::Device *realMTLDevice);
 
   // Serialised MTLDevice APIs
@@ -85,8 +103,33 @@ public:
   CaptureState &GetStateRef() { return m_State; }
   CaptureState GetState() { return m_State; }
   MetalResourceManager *GetResourceManager() { return m_ResourceManager; };
+  void WaitForGPU();
   WriteSerialiser &GetThreadSerialiser();
 
+  // IFrameCapturer interface
+  RDCDriver GetFrameCaptureDriver() { return RDCDriver::Metal; }
+  void StartFrameCapture(DeviceOwnedWindow devWnd);
+  bool EndFrameCapture(DeviceOwnedWindow devWnd);
+  bool DiscardFrameCapture(DeviceOwnedWindow devWnd);
+  // IFrameCapturer interface
+
+  void CaptureCmdBufCommit(MetalResourceRecord *record);
+  void CaptureCmdBufEnqueue(MetalResourceRecord *record);
+
+  void AddFrameCaptureRecordChunk(Chunk *chunk) { m_FrameCaptureRecord->AddChunk(chunk); }
+  // From ResourceManager interface
+  bool Prepare_InitialState(WrappedMTLObject *res);
+  uint64_t GetSize_InitialState(ResourceId id, const MetalInitialContents &initial);
+  template <typename SerialiserType>
+  bool Serialise_InitialState(SerialiserType &ser, ResourceId id, MetalResourceRecord *record,
+                              const MetalInitialContents *initial);
+  void Create_InitialState(ResourceId id, WrappedMTLObject *live, bool hasData);
+  void Apply_InitialState(WrappedMTLObject *live, const MetalInitialContents &initial);
+  // From ResourceManager interface
+
+  void RegisterMetalLayer(CA::MetalLayer *mtlLayer);
+  void UnregisterMetalLayer(CA::MetalLayer *mtlLayer);
+
   enum
   {
     TypeEnum = eResDevice
@@ -98,13 +141,18 @@ public:
 private:
   static void MTLFixupForMetalDriverAssert();
   static void MTLHookObjcMethods();
-  bool Prepare_InitialState(WrappedMTLObject *res);
-  uint64_t GetSize_InitialState(ResourceId id, const MetalInitialContents &initial);
+  void FirstFrame();
+  void AdvanceFrame();
+  void Present(MetalResourceRecord *record);
+
+  void CaptureClearSubmittedCmdBuffers();
+  void CaptureCmdBufSubmit(MetalResourceRecord *record);
+  void EndCaptureFrame();
+
   template <typename SerialiserType>
-  bool Serialise_InitialState(SerialiserType &ser, ResourceId id, MetalResourceRecord *record,
-                              const MetalInitialContents *initial);
-  void Create_InitialState(ResourceId id, WrappedMTLObject *live, bool hasData);
-  void Apply_InitialState(WrappedMTLObject *live, const MetalInitialContents &initial);
+  bool Serialise_CaptureScope(SerialiserType &ser);
+  template <typename SerialiserType>
+  bool Serialise_BeginCaptureFrame(SerialiserType &ser);
 
   WrappedMTLTexture *Common_NewTexture(RDMTL::TextureDescriptor &descriptor, MetalChunk chunkType,
                                        bool ioSurfaceTexture, IOSurfaceRef iosurface,
@@ -112,15 +160,50 @@ private:
   WrappedMTLBuffer *Common_NewBuffer(bool withBytes, const void *pointer, NS::UInteger length,
                                      MTL::ResourceOptions options);
 
-  MetalResourceManager *m_ResourceManager;
+  MetalResourceManager *m_ResourceManager = NULL;
 
   // Back buffer and swap chain emulation
-  Threading::CriticalSection m_PotentialBackBuffersLock;
-  std::unordered_set<WrappedMTLTexture *> m_PotentialBackBuffers;
+  Threading::CriticalSection m_CapturePotentialBackBuffersLock;
+  std::unordered_set<WrappedMTLTexture *> m_CapturePotentialBackBuffers;
+  Threading::CriticalSection m_CaptureOutputLayersLock;
+  std::unordered_set<CA::MetalLayer *> m_CaptureOutputLayers;
 
   CaptureState m_State;
+  bool m_AppControlledCapture = false;
 
   uint64_t threadSerialiserTLSSlot;
   Threading::CriticalSection m_ThreadSerialisersLock;
   rdcarray<WriteSerialiser *> m_ThreadSerialisers;
+  uint64_t m_SectionVersion = 0;
+
+  MetalCapturer m_Capturer;
+  uint32_t m_FrameCounter = 0;
+  rdcarray<FrameDescription> m_CapturedFrames;
+  Threading::RWLock m_CapTransitionLock;
+  MetalResourceRecord *m_FrameCaptureRecord = NULL;
+
+  // record the command buffer records to insert them individually
+  // (even if they were recorded locklessly in parallel)
+  // queue submit order will enforce/display ordering, record order is not important
+  Threading::CriticalSection m_CaptureCommandBuffersLock;
+  rdcarray<MetalResourceRecord *> m_CaptureCommandBuffersEnqueued;
+  rdcarray<MetalResourceRecord *> m_CaptureCommandBuffersSubmitted;
+
+  PerformanceTimer m_CaptureTimer;
+  MetalInitParams m_InitParams;
+
+  MTL::CommandQueue *m_mtlCommandQueue = NULL;
 };
+
+inline void MetalCapturer::StartFrameCapture(DeviceOwnedWindow devWnd)
+{
+  return m_Device.StartFrameCapture(devWnd);
+}
+inline bool MetalCapturer::EndFrameCapture(DeviceOwnedWindow devWnd)
+{
+  return m_Device.EndFrameCapture(devWnd);
+}
+inline bool MetalCapturer::DiscardFrameCapture(DeviceOwnedWindow devWnd)
+{
+  return m_Device.DiscardFrameCapture(devWnd);
+}

renderdoc/driver/metal/metal_helpers_bridge.mm

@@ -104,3 +104,19 @@ CA::MetalDrawable *ObjC::CAMetalLayer_nextDrawable(void *layerHandle)
   CA::MetalDrawable *drawable = (__bridge CA::MetalDrawable *)[metalLayer nextDrawable];
   return drawable;
 }
+
+@interface ObjCTrackedCAMetalLayer : NSObject
+@end
+
+@implementation ObjCTrackedCAMetalLayer
+
+// Silence compiler warning
+// error: method possibly missing a [super dealloc] call [-Werror,-Wobjc-missing-super-calls]
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wobjc-missing-super-calls"
+- (void)dealloc
+{
+  ((TrackedCAMetalLayer *)self)->StopTracking();
+}
+#pragma clang diagnostic pop
+@end

renderdoc/driver/metal/metal_init_state.cpp

@@ -62,9 +62,5 @@ void WrappedMTLDevice::Apply_InitialState(WrappedMTLObject *live, const MetalIni
   METAL_NOT_IMPLEMENTED();
 }
 
-template bool WrappedMTLDevice::Serialise_InitialState(ReadSerialiser &ser, ResourceId id,
-                                                       MetalResourceRecord *record,
-                                                       const MetalInitialContents *initial);
-template bool WrappedMTLDevice::Serialise_InitialState(WriteSerialiser &ser, ResourceId id,
-                                                       MetalResourceRecord *record,
-                                                       const MetalInitialContents *initial);
+INSTANTIATE_FUNCTION_SERIALISED(WrappedMTLDevice, void, InitialState, ResourceId id,
+                                MetalResourceRecord *record, const MetalInitialContents *initial);

renderdoc/driver/metal/metal_resources.h

@@ -122,17 +122,14 @@ inline MTL::Resource *Unwrap(WrappedMTLResource *obj)
   return Unwrap<MTL::Resource *>((WrappedMTLObject *)obj);
 }
 
-enum class MetalCmdBufferStatus : uint32_t
+enum class MetalCmdBufferStatus : uint8_t
 {
-  NoFlags = 0,
-  Enqueued = 1 << 0,
-  Committed = 1 << 1,
-  Submitted = 1 << 2,
-  Presented = 1 << 3,
+  Unknown,
+  Enqueued,
+  Committed,
+  Submitted,
 };
 
-BITMASK_OPERATORS(MetalCmdBufferStatus);
-
 struct MetalCmdBufferRecordingInfo
 {
   MetalCmdBufferRecordingInfo(WrappedMTLCommandQueue *parentQueue) : queue(parentQueue) {}
@@ -147,7 +144,8 @@ struct MetalCmdBufferRecordingInfo
   CA::MetalLayer *outputLayer = NULL;
   // The texture to present
   WrappedMTLTexture *backBuffer = NULL;
-  MetalCmdBufferStatus flags = MetalCmdBufferStatus::NoFlags;
+  MetalCmdBufferStatus status = MetalCmdBufferStatus::Unknown;
+  bool presented = false;
 };
 
 struct MetalResourceRecord : public ResourceRecord

renderdoc/driver/metal/metal_stringise.cpp

@@ -1149,13 +1149,12 @@ rdcstr DoStringise(const MetalResourceType &el)
 template <>
 rdcstr DoStringise(const MetalCmdBufferStatus &el)
 {
-  BEGIN_BITFIELD_STRINGISE(MetalCmdBufferStatus)
+  BEGIN_ENUM_STRINGISE(MetalCmdBufferStatus)
   {
-    STRINGISE_BITFIELD_CLASS_VALUE(NoFlags);
-    STRINGISE_BITFIELD_CLASS_BIT(Enqueued);
-    STRINGISE_BITFIELD_CLASS_BIT(Committed);
-    STRINGISE_BITFIELD_CLASS_BIT(Submitted);
-    STRINGISE_BITFIELD_CLASS_BIT(Presented);
+    STRINGISE_ENUM_CLASS(Unknown);
+    STRINGISE_ENUM_CLASS(Enqueued);
+    STRINGISE_ENUM_CLASS(Committed);
+    STRINGISE_ENUM_CLASS(Submitted);
   }
-  END_BITFIELD_STRINGISE()
+  END_ENUM_STRINGISE()
 }

renderdoc/driver/metal/metal_types.cpp

@@ -75,6 +75,36 @@ RDCCOMPILE_ASSERT(sizeof(NS::UInteger) == sizeof(std::uintptr_t),
 METALCPP_WRAPPED_PROTOCOLS(DEFINE_OBJC_HELPERS)
 #undef DEFINE_OBJC_HELPERS
 
+TrackedCAMetalLayer::TrackedCAMetalLayer(CA::MetalLayer *mtlLayer, WrappedMTLDevice *device)
+{
+  m_mtlLayer = mtlLayer;
+  m_Device = device;
+
+  RDCCOMPILE_ASSERT((offsetof(TrackedCAMetalLayer, m_ObjcBridge) == 0),
+                    "m_ObjcBridge must be at offsetof 0");
+  const char *const className = "ObjCTrackedCAMetalLayer";
+  static Class klass = objc_lookUpClass(className);
+  static size_t classSize = class_getInstanceSize(klass);
+  if(classSize != sizeof(m_ObjcBridge))
+  {
+    RDCFATAL("'%s' classSize != sizeof(m_ObjcBridge) %lu != %lu", className, classSize,
+             sizeof(m_ObjcBridge));
+  }
+  id objc = objc_constructInstance(klass, &m_ObjcBridge);
+  if(objc != (id)&m_ObjcBridge)
+  {
+    RDCFATAL("'%s' objc != m_ObjcBridge %p != %p", className, objc, &m_ObjcBridge);
+  }
+  objc_setAssociatedObject((id)m_mtlLayer, objc, objc, OBJC_ASSOCIATION_RETAIN);
+  ((NS::Object *)objc)->release();
+}
+
+void TrackedCAMetalLayer::StopTracking()
+{
+  m_Device->UnregisterMetalLayer(m_mtlLayer);
+  delete this;
+}
+
 namespace RDMTL
 {
 static bool ValidData(MTL::VertexAttributeDescriptor *attribute)

renderdoc/driver/metal/metal_types.h

@@ -91,6 +91,26 @@ METALCPP_WRAPPED_PROTOCOLS(DECLARE_OBJC_HELPERS)
 METALCPP_UNIMPLEMENTED_WRAPPED_PROTOCOLS(DECLARE_UNIMPLEMENTED_WRAPPED_CPP_HELPERS)
 #undef DECLARE_UNIMPLEMENTED_WRAPPED_CPP_HELPERS
 
+class TrackedCAMetalLayer
+{
+public:
+  TrackedCAMetalLayer() = delete;
+  ~TrackedCAMetalLayer() = default;
+
+  static void Track(CA::MetalLayer *mtlLayer, WrappedMTLDevice *device)
+  {
+    new TrackedCAMetalLayer(mtlLayer, device);
+  }
+  void StopTracking();
+
+private:
+  TrackedCAMetalLayer(CA::MetalLayer *real, WrappedMTLDevice *device);
+
+  void *m_ObjcBridge = NULL;
+  WrappedMTLDevice *m_Device = NULL;
+  CA::MetalLayer *m_mtlLayer = NULL;
+};
+
 #define MTL_DECLARE_REFLECTION_TYPE(TYPE)        \
   template <>                                    \
   inline rdcliteral TypeName<MTL::TYPE>()        \

renderdoccmd/renderdoccmd_apple.cpp

@@ -113,6 +113,11 @@ int main(int argc, char *argv[])
     count++;
 #endif
 
+#if defined(RENDERDOC_SUPPORT_METAL)
+    support += "Metal, ";
+    count++;
+#endif
+
     if(count == 0)
     {
       support += "None.";