From 9849fcd7f9fb24bd0c9e948530a94724b7039d05 Mon Sep 17 00:00:00 2001
From: baldurk <baldurk@baldurk.org>
Date: Mon, 1 Mar 2021 11:44:37 +0000
Subject: [PATCH] Handle AS/MS subobjects explicitly, ignore empty ones. Closes
 #2193

* If we get passed an empty/degenerate AS or MS we can ignore it, the
  application is responsible for ensuring that it doesn't pass any *actual*
  shaders here.
---
 renderdoc/driver/d3d12/d3d12_common.cpp       | 20 +++++++++++++++++++
 renderdoc/driver/d3d12/d3d12_common.h         |  2 ++
 renderdoc/driver/d3d12/d3d12_device_wrap2.cpp |  3 +++
 3 files changed, 25 insertions(+)

diff --git a/renderdoc/driver/d3d12/d3d12_common.cpp b/renderdoc/driver/d3d12/d3d12_common.cpp
index 1c6c23797..d0e68b746 100644
--- a/renderdoc/driver/d3d12/d3d12_common.cpp
+++ b/renderdoc/driver/d3d12/d3d12_common.cpp
@@ -1167,6 +1167,26 @@ D3D12_EXPANDED_PIPELINE_STATE_STREAM_DESC::D3D12_EXPANDED_PIPELINE_STATE_STREAM_
         ITER_ADV(D3D12_VIEW_INSTANCING_DESC);
         break;
       }
+      case D3D12_PIPELINE_STATE_SUBOBJECT_TYPE_AS:
+      {
+        if(ptr->data.shader.BytecodeLength > 0)
+        {
+          RDCERR("AS passed to D3D12_PIPELINE_STATE_STREAM_DESC but mesh shaders not supported");
+          errored = true;
+        }
+        ITER_ADV(D3D12_SHADER_BYTECODE);
+        break;
+      }
+      case D3D12_PIPELINE_STATE_SUBOBJECT_TYPE_MS:
+      {
+        if(ptr->data.shader.BytecodeLength > 0)
+        {
+          RDCERR("MS passed to D3D12_PIPELINE_STATE_STREAM_DESC but mesh shaders not supported");
+          errored = true;
+        }
+        ITER_ADV(D3D12_SHADER_BYTECODE);
+        break;
+      }
       default:
       {
         RDCERR("Unknown subobject type %d", obj->type);
diff --git a/renderdoc/driver/d3d12/d3d12_common.h b/renderdoc/driver/d3d12/d3d12_common.h
index 406f954cf..c1a566ca0 100644
--- a/renderdoc/driver/d3d12/d3d12_common.h
+++ b/renderdoc/driver/d3d12/d3d12_common.h
@@ -459,6 +459,8 @@ struct D3D12_EXPANDED_PIPELINE_STATE_STREAM_DESC
   // construct from the stream descriptor
   D3D12_EXPANDED_PIPELINE_STATE_STREAM_DESC(const D3D12_PIPELINE_STATE_STREAM_DESC &stream);
 
+  bool errored = false;
+
   // graphics properties
   ID3D12RootSignature *pRootSignature = NULL;
   D3D12_SHADER_BYTECODE VS = {};
diff --git a/renderdoc/driver/d3d12/d3d12_device_wrap2.cpp b/renderdoc/driver/d3d12/d3d12_device_wrap2.cpp
index 773acb8d4..7c20f4ae6 100644
--- a/renderdoc/driver/d3d12/d3d12_device_wrap2.cpp
+++ b/renderdoc/driver/d3d12/d3d12_device_wrap2.cpp
@@ -194,6 +194,9 @@ HRESULT WrappedID3D12Device::CreatePipelineState(const D3D12_PIPELINE_STATE_STRE
   D3D12_PACKED_PIPELINE_STATE_STREAM_DESC unwrappedDesc = expandedDesc;
   unwrappedDesc.Unwrap();
 
+  if(expandedDesc.errored)
+    return E_INVALIDARG;
+
   if(ppPipelineState == NULL)
     return m_pDevice3->CreatePipelineState(unwrappedDesc.AsDescStream(), riid, ppPipelineState);