Change verify map writes capture option to verify buffer access.

* This option will now toggle on the behaviour to fill undefined buffer contents
  with a marker value, both if they're created without data (it will be zero
  filled instead) or mapped with discard (it will keep the old contents
  instead).
* There were too many hard to find problems or misconceptions about the buffer
  filling for it to be useful. Now it will be opt-in instead.

renderdoc/api/app/renderdoc_app.h

@@ -126,15 +126,24 @@ typedef enum RENDERDOC_CaptureOption {
   //
   eRENDERDOC_Option_DelayForDebugger = 5,
 
-  // Verify any writes to mapped buffers, by checking the memory after the
-  // bounds of the returned pointer to detect any modification.
+  // Verify buffer access. This includes checking the memory returned by a Map() call to
+  // detect any out-of-bounds modification, as well as initialising buffers with undefined contents
+  // to a marker value to catch use of uninitialised memory.
+  //
+  // NOTE: This option is only valid for OpenGL and D3D11. Explicit APIs such as D3D12 and Vulkan do
+  // not do the same kind of interception & checking and undefined contents are really undefined.
   //
   // Default - disabled
   //
-  // 1 - Verify any writes to mapped buffers
-  // 0 - No verification is performed, and overwriting bounds may cause
-  //     crashes or corruption in RenderDoc
-  eRENDERDOC_Option_VerifyMapWrites = 6,
+  // 1 - Verify buffer access
+  // 0 - No verification is performed, and overwriting bounds may cause crashes or corruption in
+  //     RenderDoc.
+  eRENDERDOC_Option_VerifyBufferAccess = 6,
+
+  // The old name for eRENDERDOC_Option_VerifyBufferAccess was eRENDERDOC_Option_VerifyMapWrites.
+  // This option now controls the filling of uninitialised buffers with 0xdddddddd which was
+  // previously always enabled
+  eRENDERDOC_Option_VerifyMapWrites = eRENDERDOC_Option_VerifyBufferAccess,
 
   // Hooks any system API calls that create child processes, and injects
   // RenderDoc into them recursively with the same options.
@@ -177,7 +186,7 @@ typedef enum RENDERDOC_CaptureOption {
   // and replayed many times will not be available and may cause a failure to
   // capture.
   //
-  // Note this is only true for APIs where multithreading is difficult or
+  // NOTE: This is only true for APIs where multithreading is difficult or
   // discouraged. Newer APIs like Vulkan and D3D12 will ignore this option
   // and always capture all command lists since the API is heavily oriented
   // around it and the overheads have been reduced by API design.
@@ -525,6 +534,7 @@ typedef enum RENDERDOC_Version {
   eRENDERDOC_API_Version_1_1_1 = 10101,    // RENDERDOC_API_1_1_1 = 1 01 01
   eRENDERDOC_API_Version_1_1_2 = 10102,    // RENDERDOC_API_1_1_2 = 1 01 02
   eRENDERDOC_API_Version_1_2_0 = 10200,    // RENDERDOC_API_1_2_0 = 1 02 00
+  eRENDERDOC_API_Version_1_3_0 = 10300,    // RENDERDOC_API_1_3_0 = 1 03 00
 } RENDERDOC_Version;
 
 // API version changelog:
@@ -542,8 +552,14 @@ typedef enum RENDERDOC_Version {
 //         branch.
 // 1.2.0 - Added feature: SetCaptureFileComments() to add comments to a capture file that will be
 //         displayed in the UI program on load.
+// 1.3.0 - Added feature: New capture option eRENDERDOC_Option_AllowUnsupportedVendorExtensions
+//         which allows users to opt-in to allowing unsupported vendor extensions to function.
+//         Should be used at the user's own risk.
+//         Refactor: Renamed eRENDERDOC_Option_VerifyMapWrites to
+//         eRENDERDOC_Option_VerifyBufferAccess, which now also controls initialisation to
+//         0xdddddddd of uninitialised buffer contents.
 
-typedef struct RENDERDOC_API_1_2_0
+typedef struct RENDERDOC_API_1_3_0
 {
   pRENDERDOC_GetAPIVersion GetAPIVersion;
 
@@ -606,14 +622,15 @@ typedef struct RENDERDOC_API_1_2_0
 
   // new function in 1.2.0
   pRENDERDOC_SetCaptureFileComments SetCaptureFileComments;
-} RENDERDOC_API_1_2_0;
+} RENDERDOC_API_1_3_0;
 
-typedef RENDERDOC_API_1_2_0 RENDERDOC_API_1_0_0;
-typedef RENDERDOC_API_1_2_0 RENDERDOC_API_1_0_1;
-typedef RENDERDOC_API_1_2_0 RENDERDOC_API_1_0_2;
-typedef RENDERDOC_API_1_2_0 RENDERDOC_API_1_1_0;
-typedef RENDERDOC_API_1_2_0 RENDERDOC_API_1_1_1;
-typedef RENDERDOC_API_1_2_0 RENDERDOC_API_1_1_2;
+typedef RENDERDOC_API_1_3_0 RENDERDOC_API_1_0_0;
+typedef RENDERDOC_API_1_3_0 RENDERDOC_API_1_0_1;
+typedef RENDERDOC_API_1_3_0 RENDERDOC_API_1_0_2;
+typedef RENDERDOC_API_1_3_0 RENDERDOC_API_1_1_0;
+typedef RENDERDOC_API_1_3_0 RENDERDOC_API_1_1_1;
+typedef RENDERDOC_API_1_3_0 RENDERDOC_API_1_1_2;
+typedef RENDERDOC_API_1_3_0 RENDERDOC_API_1_2_0;
 
 //////////////////////////////////////////////////////////////////////////////////////////////////
 // RenderDoc API entry point

renderdoc/api/replay/capture_options.h

@@ -131,17 +131,23 @@ Default - 0 seconds
 )");
   uint32_t delayForDebugger;
 
-  DOCUMENT(R"(Verify any writes to mapped buffers, by checking the memory after the
-bounds of the returned pointer to detect any modification.
+  DOCUMENT(R"(Verify buffer access. This includes checking the memory returned by a Map() call to
+detect any out-of-bounds modification, as well as initialising buffers with undefined contents to
+a marker value to catch use of uninitialised memory.
+
+.. note::
+
+  This option is only valid for OpenGL and D3D11. Explicit APIs such as D3D12 and Vulkan do not
+  do the same kind of interception & checking and undefined contents are really undefined.
 
 Default - disabled
 
-``True`` - Verify any writes to mapped buffers.
+``True`` - Verify buffer access.
 
-``False`` - No verification is performed, and overwriting bounds may cause
-crashes or corruption in RenderDoc.
+``False`` - No verification is performed, and overwriting bounds may cause crashes or corruption in
+RenderDoc.
 )");
-  bool verifyMapWrites;
+  bool verifyBufferAccess;
 
   DOCUMENT(R"(Hooks any system API calls that create child processes, and injects
 RenderDoc into them recursively with the same options.

renderdoc/driver/d3d11/d3d11_context_wrap.cpp

@@ -7237,12 +7237,13 @@ bool WrappedID3D11DeviceContext::Serialise_Map(SerialiserType &ser, ID3D11Resour
 
     if(MapType == D3D11_MAP_WRITE_DISCARD)
     {
-      memset(appMem, 0xcc, mapLength);
+      if(RenderDoc::Inst().GetCaptureOptions().verifyBufferAccess)
+        memset(appMem, 0xcc, mapLength);
       memcpy(record->GetShadowPtr(ctxMapID, 1), appMem, mapLength);
     }
 
     intercept = MapIntercept();
-    intercept.verifyWrite = (RenderDoc::Inst().GetCaptureOptions().verifyMapWrites != 0);
+    intercept.verifyWrite = RenderDoc::Inst().GetCaptureOptions().verifyBufferAccess;
     intercept.SetD3D(mappedResource);
     intercept.InitWrappedResource(resMap, Subresource, appMem);
     intercept.MapType = MapType;
@@ -7260,7 +7261,7 @@ bool WrappedID3D11DeviceContext::Serialise_Map(SerialiserType &ser, ID3D11Resour
     mapLength = (size_t)record->Length;
 
     intercept = MapIntercept();
-    intercept.verifyWrite = (RenderDoc::Inst().GetCaptureOptions().verifyMapWrites != 0);
+    intercept.verifyWrite = RenderDoc::Inst().GetCaptureOptions().verifyBufferAccess;
     intercept.SetD3D(mappedResource);
     intercept.MapType = MapType;
     intercept.MapFlags = MapFlags;
@@ -7411,7 +7412,7 @@ HRESULT WrappedID3D11DeviceContext::Map(ID3D11Resource *pResource, UINT Subresou
 
       record->UpdateCount++;
 
-      if(record->UpdateCount > 60 && RenderDoc::Inst().GetCaptureOptions().verifyMapWrites == 0)
+      if(record->UpdateCount > 60 && !RenderDoc::Inst().GetCaptureOptions().verifyBufferAccess)
       {
         m_HighTrafficResources.insert(Id);
         MarkDirtyResource(Id);

renderdoc/driver/d3d11/d3d11_device_wrap.cpp

@@ -45,8 +45,10 @@ bool WrappedID3D11Device::Serialise_CreateBuffer(SerialiserType &ser, const D3D1
   {
     fakeData.pSysMem = new byte[Descriptor.ByteWidth];
     fakeData.SysMemPitch = fakeData.SysMemSlicePitch = Descriptor.ByteWidth;
-    // fill with 0xfefefefe to indicate that the data is uninitialised.
-    memset((void *)fakeData.pSysMem, 0xfe, Descriptor.ByteWidth);
+    // fill with 0xdddddddd to indicate that the data is uninitialised, if that option is enabled
+    memset((void *)fakeData.pSysMem,
+           RenderDoc::Inst().GetCaptureOptions().verifyBufferAccess ? 0xdd : 0x0,
+           Descriptor.ByteWidth);
     pInitialData = &fakeData;
   }
 

renderdoc/driver/gl/wrappers/gl_buffer_funcs.cpp

@@ -479,7 +479,7 @@ void WrappedOpenGL::glNamedBufferStorageEXT(GLuint buffer, GLsizeiptr size, cons
   if(IsCaptureMode(m_State) && data == NULL)
   {
     dummy = new byte[size];
-    memset(dummy, 0xdd, size);
+    memset(dummy, RenderDoc::Inst().GetCaptureOptions().verifyBufferAccess ? 0xdd : 0x0, size);
     data = dummy;
 
     GLResourceRecord *record = GetResourceManager()->GetResourceRecord(BufferRes(GetCtx(), buffer));
@@ -509,7 +509,7 @@ void WrappedOpenGL::glBufferStorage(GLenum target, GLsizeiptr size, const void *
   if(IsCaptureMode(m_State) && data == NULL)
   {
     dummy = new byte[size];
-    memset(dummy, 0xdd, size);
+    memset(dummy, RenderDoc::Inst().GetCaptureOptions().verifyBufferAccess ? 0xdd : 0x0, size);
     data = dummy;
 
     GLResourceRecord *record = GetCtxData().m_BufferRecord[BufferIdx(target)];
@@ -573,7 +573,7 @@ void WrappedOpenGL::glNamedBufferDataEXT(GLuint buffer, GLsizeiptr size, const v
   if(IsCaptureMode(m_State) && data == NULL)
   {
     dummy = new byte[size];
-    memset(dummy, 0xdd, size);
+    memset(dummy, RenderDoc::Inst().GetCaptureOptions().verifyBufferAccess ? 0xdd : 0x0, size);
     data = dummy;
 
     GLResourceRecord *record = GetResourceManager()->GetResourceRecord(BufferRes(GetCtx(), buffer));
@@ -714,7 +714,7 @@ void WrappedOpenGL::glBufferData(GLenum target, GLsizeiptr size, const void *dat
   if(IsCaptureMode(m_State) && data == NULL)
   {
     dummy = new byte[size];
-    memset(dummy, 0xdd, size);
+    memset(dummy, RenderDoc::Inst().GetCaptureOptions().verifyBufferAccess ? 0xdd : 0x0, size);
     data = dummy;
 
     GLResourceRecord *record = GetCtxData().m_BufferRecord[idx];
@@ -740,10 +740,9 @@ void WrappedOpenGL::glBufferData(GLenum target, GLsizeiptr size, const void *dat
     if(IsBackgroundCapturing(m_State) && record->HasDataPtr() &&
        size == (GLsizeiptr)record->Length && usage == record->usage)
     {
-      if(data)
-        memcpy(record->GetDataPtr(), data, (size_t)size);
-      else
-        memset(record->GetDataPtr(), 0xbe, (size_t)size);
+      // if data was NULL, it was set to dummy above.
+      RDCASSERT(data);
+      memcpy(record->GetDataPtr(), data, (size_t)size);
 
       SAFE_DELETE_ARRAY(dummy);
 
@@ -1934,7 +1933,7 @@ void *WrappedOpenGL::glMapNamedBufferRangeEXT(GLuint buffer, GLintptr offset, GL
     if(record->Map.persistentPtr)
       directMap = false;
 
-    bool verifyWrite = (RenderDoc::Inst().GetCaptureOptions().verifyMapWrites != 0);
+    bool verifyWrite = RenderDoc::Inst().GetCaptureOptions().verifyBufferAccess;
 
     // must also intercept to verify writes
     if(verifyWrite && (access & GL_MAP_WRITE_BIT))
@@ -2004,8 +2003,8 @@ void *WrappedOpenGL::glMapNamedBufferRangeEXT(GLuint buffer, GLintptr offset, GL
         // comparison & modified buffer in case the application calls glMemoryBarrier(..) at any
         // time.
 
-        // if we're invalidating, mark the whole range as 0xcc
-        if(invalidateMap)
+        // if we're invalidating and verifying, mark the whole range as 0xcc
+        if(invalidateMap && RenderDoc::Inst().GetCaptureOptions().verifyBufferAccess)
         {
           memset(record->GetShadowPtr(0) + offset, 0xcc, length);
           memset(record->GetShadowPtr(1) + offset, 0xcc, length);
@@ -2050,7 +2049,7 @@ void *WrappedOpenGL::glMapNamedBufferRangeEXT(GLuint buffer, GLintptr offset, GL
         }
 
         // if we're invalidating, mark the whole range as 0xcc
-        if(invalidateMap)
+        if(invalidateMap && RenderDoc::Inst().GetCaptureOptions().verifyBufferAccess)
         {
           memset(shadow + offset, 0xcc, length);
           memset(record->GetShadowPtr(1) + offset, 0xcc, length);
@@ -2077,10 +2076,10 @@ void *WrappedOpenGL::glMapNamedBufferRangeEXT(GLuint buffer, GLintptr offset, GL
           }
 
           // if we're not invalidating, we need the existing contents
-          if(!invalidateMap)
-            memcpy(shadow, record->GetDataPtr(), buflength);
-          else
+          if(invalidateMap && RenderDoc::Inst().GetCaptureOptions().verifyBufferAccess)
             memset(shadow + offset, 0xcc, length);
+          else
+            memcpy(shadow, record->GetDataPtr(), buflength);
 
           ptr = shadow;
         }

renderdoc/replay/app_api.cpp

@@ -239,22 +239,22 @@ int RENDERDOC_CC SetCaptureOptionF32(RENDERDOC_CaptureOption opt, float val);
 uint32_t RENDERDOC_CC GetCaptureOptionU32(RENDERDOC_CaptureOption opt);
 float RENDERDOC_CC GetCaptureOptionF32(RENDERDOC_CaptureOption opt);
 
-void RENDERDOC_CC GetAPIVersion_1_2_0(int *major, int *minor, int *patch)
+void RENDERDOC_CC GetAPIVersion_1_3_0(int *major, int *minor, int *patch)
 {
   if(major)
     *major = 1;
   if(minor)
-    *minor = 2;
+    *minor = 3;
   if(patch)
     *patch = 0;
 }
 
-RENDERDOC_API_1_2_0 api_1_2_0;
-void Init_1_2_0()
+RENDERDOC_API_1_3_0 api_1_3_0;
+void Init_1_3_0()
 {
-  RENDERDOC_API_1_2_0 &api = api_1_2_0;
+  RENDERDOC_API_1_3_0 &api = api_1_3_0;
 
-  api.GetAPIVersion = &GetAPIVersion_1_2_0;
+  api.GetAPIVersion = &GetAPIVersion_1_3_0;
 
   api.SetCaptureOptionU32 = &SetCaptureOptionU32;
   api.SetCaptureOptionF32 = &SetCaptureOptionF32;
@@ -317,13 +317,14 @@ extern "C" RENDERDOC_API int RENDERDOC_CC RENDERDOC_GetAPI(RENDERDOC_Version ver
     ret = 1;                                                       \
   }
 
-  API_VERSION_HANDLE(1_0_0, 1_2_0);
-  API_VERSION_HANDLE(1_0_1, 1_2_0);
-  API_VERSION_HANDLE(1_0_2, 1_2_0);
-  API_VERSION_HANDLE(1_1_0, 1_2_0);
-  API_VERSION_HANDLE(1_1_1, 1_2_0);
-  API_VERSION_HANDLE(1_1_2, 1_2_0);
-  API_VERSION_HANDLE(1_2_0, 1_2_0);
+  API_VERSION_HANDLE(1_0_0, 1_3_0);
+  API_VERSION_HANDLE(1_0_1, 1_3_0);
+  API_VERSION_HANDLE(1_0_2, 1_3_0);
+  API_VERSION_HANDLE(1_1_0, 1_3_0);
+  API_VERSION_HANDLE(1_1_1, 1_3_0);
+  API_VERSION_HANDLE(1_1_2, 1_3_0);
+  API_VERSION_HANDLE(1_2_0, 1_3_0);
+  API_VERSION_HANDLE(1_3_0, 1_3_0);
 
 #undef API_VERSION_HANDLE
 

renderdoc/replay/capture_options.cpp

@@ -42,7 +42,7 @@ int RENDERDOC_CC SetCaptureOptionU32(RENDERDOC_CaptureOption opt, uint32_t val)
       opts.captureCallstacksOnlyDraws = (val != 0);
       break;
     case eRENDERDOC_Option_DelayForDebugger: opts.delayForDebugger = val; break;
-    case eRENDERDOC_Option_VerifyMapWrites: opts.verifyMapWrites = (val != 0); break;
+    case eRENDERDOC_Option_VerifyBufferAccess: opts.verifyBufferAccess = (val != 0); break;
     case eRENDERDOC_Option_HookIntoChildren: opts.hookIntoChildren = (val != 0); break;
     case eRENDERDOC_Option_RefAllResources: opts.refAllResources = (val != 0); break;
     case eRENDERDOC_Option_SaveAllInitials:
@@ -77,7 +77,7 @@ int RENDERDOC_CC SetCaptureOptionF32(RENDERDOC_CaptureOption opt, float val)
       opts.captureCallstacksOnlyDraws = (val != 0.0f);
       break;
     case eRENDERDOC_Option_DelayForDebugger: opts.delayForDebugger = (uint32_t)val; break;
-    case eRENDERDOC_Option_VerifyMapWrites: opts.verifyMapWrites = (val != 0.0f); break;
+    case eRENDERDOC_Option_VerifyBufferAccess: opts.verifyBufferAccess = (val != 0.0f); break;
     case eRENDERDOC_Option_HookIntoChildren: opts.hookIntoChildren = (val != 0.0f); break;
     case eRENDERDOC_Option_RefAllResources: opts.refAllResources = (val != 0.0f); break;
     case eRENDERDOC_Option_SaveAllInitials:
@@ -111,8 +111,8 @@ uint32_t RENDERDOC_CC GetCaptureOptionU32(RENDERDOC_CaptureOption opt)
       return (RenderDoc::Inst().GetCaptureOptions().captureCallstacksOnlyDraws ? 1 : 0);
     case eRENDERDOC_Option_DelayForDebugger:
       return (RenderDoc::Inst().GetCaptureOptions().delayForDebugger);
-    case eRENDERDOC_Option_VerifyMapWrites:
-      return (RenderDoc::Inst().GetCaptureOptions().verifyMapWrites ? 1 : 0);
+    case eRENDERDOC_Option_VerifyBufferAccess:
+      return (RenderDoc::Inst().GetCaptureOptions().verifyBufferAccess ? 1 : 0);
     case eRENDERDOC_Option_HookIntoChildren:
       return (RenderDoc::Inst().GetCaptureOptions().hookIntoChildren ? 1 : 0);
     case eRENDERDOC_Option_RefAllResources:
@@ -148,8 +148,8 @@ float RENDERDOC_CC GetCaptureOptionF32(RENDERDOC_CaptureOption opt)
       return (RenderDoc::Inst().GetCaptureOptions().captureCallstacksOnlyDraws ? 1.0f : 0.0f);
     case eRENDERDOC_Option_DelayForDebugger:
       return (RenderDoc::Inst().GetCaptureOptions().delayForDebugger * 1.0f);
-    case eRENDERDOC_Option_VerifyMapWrites:
-      return (RenderDoc::Inst().GetCaptureOptions().verifyMapWrites ? 1.0f : 0.0f);
+    case eRENDERDOC_Option_VerifyBufferAccess:
+      return (RenderDoc::Inst().GetCaptureOptions().verifyBufferAccess ? 1.0f : 0.0f);
     case eRENDERDOC_Option_HookIntoChildren:
       return (RenderDoc::Inst().GetCaptureOptions().hookIntoChildren ? 1.0f : 0.0f);
     case eRENDERDOC_Option_RefAllResources:
@@ -177,7 +177,7 @@ CaptureOptions::CaptureOptions()
   captureCallstacks = false;
   captureCallstacksOnlyDraws = false;
   delayForDebugger = 0;
-  verifyMapWrites = false;
+  verifyBufferAccess = false;
   hookIntoChildren = false;
   refAllResources = false;
   captureAllCmdLists = false;

renderdoc/replay/renderdoc_serialise.inl

@@ -106,7 +106,7 @@ void DoSerialise(SerialiserType &ser, CaptureOptions &el)
   SERIALISE_MEMBER(captureCallstacks);
   SERIALISE_MEMBER(captureCallstacksOnlyDraws);
   SERIALISE_MEMBER(delayForDebugger);
-  SERIALISE_MEMBER(verifyMapWrites);
+  SERIALISE_MEMBER(verifyBufferAccess);
   SERIALISE_MEMBER(hookIntoChildren);
   SERIALISE_MEMBER(refAllResources);
   SERIALISE_MEMBER(captureAllCmdLists);