Implement GPU counters via timestamp query pools

renderdoc/driver/vulkan/Makefile

@@ -20,6 +20,7 @@ vk_manager.o \
 vk_resources.o \
 vk_debug.o \
 vk_replay.o \
+vk_counters.o \
 vk_info.o \
 vk_state.o \
 vk_linux.o \

renderdoc/driver/vulkan/renderdoc_vulkan.vcxproj

@@ -19,6 +19,7 @@
     </ProjectConfiguration>
   </ItemGroup>
   <ItemGroup>
+    <ClCompile Include="vk_counters.cpp" />
     <ClCompile Include="vk_dispatchtables.cpp" />
     <ClCompile Include="vk_initstate.cpp" />
     <ClCompile Include="vk_memory.cpp" />

renderdoc/driver/vulkan/renderdoc_vulkan.vcxproj.filters

@@ -79,6 +79,9 @@
     <ClCompile Include="vk_state.cpp">
       <Filter>Util</Filter>
     </ClCompile>
+    <ClCompile Include="vk_counters.cpp">
+      <Filter>Replay</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="vk_replay.h">

renderdoc/driver/vulkan/vk_core.h

@@ -508,6 +508,7 @@ public:
 
 	vector<FetchFrameRecord> &GetFrameRecord() { return m_FrameRecord; }
 	FetchAPIEvent GetEvent(uint32_t eventID);
+	uint32_t GetMaxEID() { return m_Events.back().eventID; }
 	const FetchDrawcall *GetDrawcall(uint32_t frameID, uint32_t eventID);
 
 	// return the pre-selected device and queue

renderdoc/driver/vulkan/vk_counters.cpp

@@ -0,0 +1,169 @@
+/******************************************************************************
+ * The MIT License (MIT)
+ * 
+ * Copyright (c) 2015 Baldur Karlsson
+ * 
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ * 
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ * 
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ ******************************************************************************/
+
+#include "vk_replay.h"
+#include "vk_core.h"
+#include "vk_resources.h"
+
+void VulkanReplay::PreDeviceInitCounters()
+{
+}
+
+void VulkanReplay::PostDeviceInitCounters()
+{
+}
+
+void VulkanReplay::PreDeviceShutdownCounters()
+{
+}
+
+void VulkanReplay::PostDeviceShutdownCounters()
+{
+}
+
+vector<uint32_t> VulkanReplay::EnumerateCounters()
+{
+	vector<uint32_t> ret;
+
+	ret.push_back(eCounter_EventGPUDuration);
+
+	return ret;
+}
+
+void VulkanReplay::DescribeCounter(uint32_t counterID, CounterDescription &desc)
+{
+	desc.counterID = counterID;
+
+	if(counterID == eCounter_EventGPUDuration)
+	{
+		desc.name = "GPU Duration";
+		desc.description = "Time taken for this event on the GPU, as measured by delta between two GPU timestamps, top to bottom of the pipe.";
+		desc.resultByteWidth = 8;
+		desc.resultCompType = eCompType_Double;
+		desc.units = eUnits_Seconds;
+	}
+	else
+	{
+		desc.name = "Unknown";
+		desc.description = "Unknown counter ID";
+		desc.resultByteWidth = 0;
+		desc.resultCompType = eCompType_None;
+		desc.units = eUnits_Absolute;
+	}
+}
+struct GPUTimerCallback : public DrawcallCallback
+{
+	GPUTimerCallback(WrappedVulkan *vk, VulkanReplay *rp, VkQueryPool qp)
+		: m_pDriver(vk)
+		, m_pReplay(rp)
+		, m_QueryPool(qp)
+	{ m_pDriver->SetDrawcallCB(this); }
+	~GPUTimerCallback()
+	{ m_pDriver->SetDrawcallCB(NULL); }
+
+	void PreDraw(uint32_t eid, VkCommandBuffer cmd)
+	{
+		ObjDisp(cmd)->CmdWriteTimestamp(Unwrap(cmd), VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, m_QueryPool, (uint32_t)(m_Results.size()*2 + 0));
+	}
+
+	bool PostDraw(uint32_t eid, VkCommandBuffer cmd)
+	{
+		ObjDisp(cmd)->CmdWriteTimestamp(Unwrap(cmd), VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, m_QueryPool, (uint32_t)(m_Results.size()*2 + 1));
+		m_Results.push_back(eid);
+		return false;
+	}
+
+	void PostRedraw(uint32_t eid, VkCommandBuffer cmd)
+	{
+	}
+
+	WrappedVulkan *m_pDriver;
+	VulkanReplay *m_pReplay;
+	VkQueryPool m_QueryPool;
+	vector<uint32_t> m_Results;
+};
+
+vector<CounterResult> VulkanReplay::FetchCounters(uint32_t frameID, uint32_t minEventID, uint32_t maxEventID, const vector<uint32_t> &counters)
+{
+	uint32_t maxEID = m_pDriver->GetMaxEID();
+
+	VkDevice dev = m_pDriver->GetDev();
+
+	VkQueryPoolCreateInfo poolCreateInfo = {
+		VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO, NULL,
+		0, VK_QUERY_TYPE_TIMESTAMP, maxEID*2, 0
+	};
+
+	VkQueryPool pool;
+	VkResult vkr = ObjDisp(dev)->CreateQueryPool(Unwrap(dev), &poolCreateInfo, NULL, &pool);
+	RDCASSERT(vkr == VK_SUCCESS);
+
+	VkCommandBuffer cmd = m_pDriver->GetNextCmd();
+
+	VkCommandBufferBeginInfo beginInfo = { VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, NULL, VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT };
+	
+	vkr = ObjDisp(dev)->BeginCommandBuffer(Unwrap(cmd), &beginInfo);
+	RDCASSERT(vkr == VK_SUCCESS);
+
+	ObjDisp(dev)->CmdResetQueryPool(Unwrap(cmd), pool, 0, maxEID*2);
+	
+	vkr = ObjDisp(dev)->EndCommandBuffer(Unwrap(cmd));
+	RDCASSERT(vkr == VK_SUCCESS);
+
+	GPUTimerCallback cb(m_pDriver, this, pool);
+
+	// replay the events to perform all the queries
+	m_pDriver->ReplayLog(frameID, 0, maxEID, eReplay_Full);
+
+	vector<uint64_t> m_Data;
+	m_Data.resize(cb.m_Results.size()*2);
+
+	vkr = ObjDisp(dev)->GetQueryPoolResults(Unwrap(dev), pool, 0, (uint32_t)m_Data.size(),
+	           sizeof(uint64_t)*m_Data.size(), &m_Data[0], sizeof(uint64_t),
+	           VK_QUERY_RESULT_64_BIT|VK_QUERY_RESULT_WAIT_BIT);
+	RDCASSERT(vkr == VK_SUCCESS);
+
+	ObjDisp(dev)->DestroyQueryPool(Unwrap(dev), pool, NULL);
+
+	vector<CounterResult> ret;
+
+	for(size_t i=0; i < cb.m_Results.size(); i++)
+	{
+		CounterResult result;
+
+		uint64_t delta = m_Data[i*2 + 1] - m_Data[i*2 + 0];
+		
+		result.eventID = cb.m_Results[i];
+		result.counterID = eCounter_EventGPUDuration;
+		result.value.d =
+			(double(m_pDriver->GetDeviceProps().limits.timestampPeriod) * double(delta)) // nanoseconds
+			/
+			(1000.0*1000.0*1000.0);  // to seconds
+
+		ret.push_back(result);
+	}
+
+	return ret;
+}
+

renderdoc/driver/vulkan/vk_debug.cpp

@@ -242,6 +242,8 @@ VulkanDebugManager::VulkanDebugManager(WrappedVulkan *driver, VkDevice dev)
 	m_pDriver = driver;
 	m_State = m_pDriver->GetState();
 
+	driver->GetReplay()->PostDeviceInitCounters();
+
 	m_ResourceManager = m_pDriver->GetResourceManager();
 
 	m_DescriptorPool = VK_NULL_HANDLE;

renderdoc/driver/vulkan/vk_replay.cpp

@@ -511,8 +511,12 @@ VulkanResourceManager *VulkanReplay::GetResourceManager()
 
 void VulkanReplay::Shutdown()
 {
+	PreDeviceShutdownCounters();
+
 	m_pDriver->Shutdown();
 	delete m_pDriver;
+
+	VulkanReplay::PostDeviceShutdownCounters();
 }
 
 APIProperties VulkanReplay::GetAPIProperties()
@@ -4314,23 +4318,6 @@ void VulkanReplay::RemoveReplacement(ResourceId id)
 	VULKANNOTIMP("RemoveReplacement");
 }
 
-vector<uint32_t> VulkanReplay::EnumerateCounters()
-{
-	VULKANNOTIMP("EnumerateCounters");
-	return vector<uint32_t>();
-}
-
-void VulkanReplay::DescribeCounter(uint32_t counterID, CounterDescription &desc)
-{
-	RDCUNIMPLEMENTED("DescribeCounter");
-}
-
-vector<CounterResult> VulkanReplay::FetchCounters(uint32_t frameID, uint32_t minEventID, uint32_t maxEventID, const vector<uint32_t> &counters)
-{
-	RDCUNIMPLEMENTED("FetchCounters");
-	return vector<CounterResult>();
-}
-
 void VulkanReplay::BuildTargetShader(string source, string entry, const uint32_t compileFlags, ShaderStageType type, ResourceId *id, string *errors)
 {
 	VULKANNOTIMP("BuildTargetShader");
@@ -4429,6 +4416,8 @@ ReplayCreateStatus Vulkan_CreateReplayDevice(const char *logfile, IReplayDriver
 		RDCLOG("Captured API version is not the same as RenderDoc's built version, expected %d got %d", VK_API_VERSION, initParams.APIVersion);
 		RDCLOG("This isn't a problem as this information is optional, but RenderDoc will replay with its own API version");
 	}
+	
+	VulkanReplay::PreDeviceInitCounters();
 
 	WrappedVulkan *vk = new WrappedVulkan(logfile);
 	vk->Initialise(initParams);

renderdoc/driver/vulkan/vk_replay.h

@@ -172,6 +172,13 @@ class VulkanReplay : public IReplayDriver
 		void InitCallstackResolver();
 		bool HasCallstacks();
 		Callstack::StackResolver *GetCallstackResolver();
+
+		// called before any VkDevice is created, to init any counters
+		static void PreDeviceInitCounters();
+		// called after any VkDevice is destroyed, to do corresponding shutdown of counters
+		static void PostDeviceShutdownCounters();
+		// called after the VkDevice is created, to init any counters
+		void PostDeviceInitCounters();
 	private:
 		struct OutputWindow
 		{
@@ -249,7 +256,10 @@ class VulkanReplay : public IReplayDriver
 		void CreateTexImageView(VkImageAspectFlags aspectFlags, VkImage liveIm, VulkanCreationInfo::Image &iminfo);
 
 		void FillCBufferVariables(rdctype::array<ShaderConstant>, vector<ShaderVariable> &outvars, const vector<byte> &data, size_t &offset);
-
+		
+		// called before the VkDevice is destroyed, to shutdown any counters
+		void PreDeviceShutdownCounters();
+		
 		VulkanDebugManager *GetDebugManager();
 		VulkanResourceManager *GetResourceManager();
 };