Implement proper pixel picking via RenderTexture & F32 readback

ReleaseNotes.md

@@ -41,7 +41,6 @@ On capture:
 On replay:
 
 * Only 2D non-array non-integer textures can currently be displayed.
-* Pixel values aren't fetched.
 * Thumbnail images aren't rendered (causes crashes/halts on nvidia's driver).
 * Auto texture range-fit or histogram display is not implemented.
 * Debug overlays aren't implemented.

renderdoc/driver/vulkan/vk_debug.cpp

@@ -86,7 +86,7 @@ struct stringdata
 	uint32_t str[256][4];
 };
 
-void VulkanDebugManager::GPUBuffer::Create(WrappedVulkan *driver, VkDevice dev, VkDeviceSize size)
+void VulkanDebugManager::GPUBuffer::Create(WrappedVulkan *driver, VkDevice dev, VkDeviceSize size, uint32_t flags)
 {
 	const VkLayerDispatchTable *vt = ObjDisp(dev);
 
@@ -94,10 +94,14 @@ void VulkanDebugManager::GPUBuffer::Create(WrappedVulkan *driver, VkDevice dev,
 
 	VkBufferCreateInfo bufInfo = {
 		VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, NULL,
-		size, VK_BUFFER_USAGE_TRANSFER_SOURCE_BIT|VK_BUFFER_USAGE_TRANSFER_DESTINATION_BIT, 0,
+		size, 0, 0,
 		VK_SHARING_MODE_EXCLUSIVE, 0, NULL,
 	};
 
+	bufInfo.usage |= VK_BUFFER_USAGE_TRANSFER_SOURCE_BIT;
+	bufInfo.usage |= VK_BUFFER_USAGE_TRANSFER_DESTINATION_BIT;
+	bufInfo.usage |= VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT;
+
 	VkResult vkr = vt->CreateBuffer(Unwrap(dev), &bufInfo, &buf);
 	RDCASSERT(vkr == VK_SUCCESS);
 
@@ -110,7 +114,10 @@ void VulkanDebugManager::GPUBuffer::Create(WrappedVulkan *driver, VkDevice dev,
 	// VKTODOMED maybe don't require host visible, and do map & copy?
 	VkMemoryAllocInfo allocInfo = {
 		VK_STRUCTURE_TYPE_MEMORY_ALLOC_INFO, NULL,
-		size, driver->GetUploadMemoryIndex(mrq.memoryTypeBits),
+		size,
+		(flags & eGPUBufferReadback)
+		? driver->GetReadbackMemoryIndex(mrq.memoryTypeBits)
+		: driver->GetUploadMemoryIndex(mrq.memoryTypeBits),
 	};
 
 	vkr = vt->AllocMemory(Unwrap(dev), &allocInfo, &mem);
@@ -364,7 +371,7 @@ VulkanDebugManager::VulkanDebugManager(WrappedVulkan *driver, VkDevice dev)
 
 	VKMGR()->WrapResource(Unwrap(dev), m_GenericDescSet);
 
-	m_GenericUBO.Create(driver, dev, 128);
+	m_GenericUBO.Create(driver, dev, 128, 0);
 	RDCCOMPILE_ASSERT(sizeof(genericuniforms) <= 128, "outline strip VBO size");
 
 	{
@@ -382,7 +389,7 @@ VulkanDebugManager::VulkanDebugManager(WrappedVulkan *driver, VkDevice dev)
 			0.0f, -1.0f, 0.0f, 1.0f,
 		};
 		
-		m_OutlineStripVBO.Create(driver, dev, 128);
+		m_OutlineStripVBO.Create(driver, dev, 128, 0);
 		RDCCOMPILE_ASSERT(sizeof(data) <= 128, "outline strip VBO size");
 		
 		float *mapped = (float *)m_OutlineStripVBO.Map(vt, dev);
@@ -392,15 +399,15 @@ VulkanDebugManager::VulkanDebugManager(WrappedVulkan *driver, VkDevice dev)
 		m_OutlineStripVBO.Unmap(vt, dev);
 	}
 
-	m_CheckerboardUBO.Create(driver, dev, 128);
-	m_TexDisplayUBO.Create(driver, dev, 128);
+	m_CheckerboardUBO.Create(driver, dev, 128, 0);
+	m_TexDisplayUBO.Create(driver, dev, 128, 0);
 
 	RDCCOMPILE_ASSERT(sizeof(displayuniforms) <= 128, "tex display size");
 		
-	m_TextGeneralUBO.Create(driver, dev, 128);
+	m_TextGeneralUBO.Create(driver, dev, 128, 0);
 	RDCCOMPILE_ASSERT(sizeof(fontuniforms) <= 128, "font uniforms size");
 
-	m_TextStringUBO.Create(driver, dev, 4096);
+	m_TextStringUBO.Create(driver, dev, 4096, 0);
 	RDCCOMPILE_ASSERT(sizeof(stringdata) <= 4096, "font uniforms size");
 	
 	string shaderSources[] = {
@@ -749,7 +756,7 @@ VulkanDebugManager::VulkanDebugManager(WrappedVulkan *driver, VkDevice dev)
 			vt->UnmapMemory(Unwrap(dev), Unwrap(m_TextAtlasMem));
 		}
 
-		m_TextGlyphUBO.Create(driver, dev, 4096);
+		m_TextGlyphUBO.Create(driver, dev, 4096, 0);
 		RDCCOMPILE_ASSERT(sizeof(Vec4f)*2*(numChars+1) < 4096, "font uniform size");
 
 		Vec4f *glyphData = (Vec4f *)m_TextGlyphUBO.Map(vt, dev);
@@ -768,6 +775,147 @@ VulkanDebugManager::VulkanDebugManager(WrappedVulkan *driver, VkDevice dev)
 		m_TextGlyphUBO.Unmap(vt, dev);
 	}
 
+	// pick pixel data
+	{
+		// create image
+		
+		VkImageCreateInfo imInfo = {
+			VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, NULL,
+			VK_IMAGE_TYPE_2D, VK_FORMAT_R32G32B32A32_SFLOAT,
+			{ 1, 1, 1 }, 1, 1, 1,
+			VK_IMAGE_TILING_OPTIMAL,
+			VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT|VK_IMAGE_USAGE_TRANSFER_SOURCE_BIT,
+			0,
+			VK_SHARING_MODE_EXCLUSIVE,
+			0, NULL,
+			VK_IMAGE_LAYOUT_UNDEFINED,
+		};
+		
+		vkr = vt->CreateImage(Unwrap(dev), &imInfo, &m_PickPixelImage);
+		RDCASSERT(vkr == VK_SUCCESS);
+
+		VKMGR()->WrapResource(Unwrap(dev), m_PickPixelImage);
+
+		VkMemoryRequirements mrq;
+		vkr = vt->GetImageMemoryRequirements(Unwrap(dev), Unwrap(m_PickPixelImage), &mrq);
+		RDCASSERT(vkr == VK_SUCCESS);
+
+		VkImageSubresource subr = { VK_IMAGE_ASPECT_COLOR, 0, 0 };
+		VkSubresourceLayout layout = { 0 };
+		vt->GetImageSubresourceLayout(Unwrap(dev), Unwrap(m_PickPixelImage), &subr, &layout);
+
+		// allocate readback memory
+		VkMemoryAllocInfo allocInfo = {
+			VK_STRUCTURE_TYPE_MEMORY_ALLOC_INFO, NULL,
+			mrq.size, driver->GetGPULocalMemoryIndex(mrq.memoryTypeBits),
+		};
+
+		vkr = vt->AllocMemory(Unwrap(dev), &allocInfo, &m_PickPixelImageMem);
+		RDCASSERT(vkr == VK_SUCCESS);
+
+		VKMGR()->WrapResource(Unwrap(dev), m_PickPixelImageMem);
+
+		vkr = vt->BindImageMemory(Unwrap(dev), Unwrap(m_PickPixelImage), Unwrap(m_PickPixelImageMem), 0);
+		RDCASSERT(vkr == VK_SUCCESS);
+
+		VkImageViewCreateInfo viewInfo = {
+			VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, NULL,
+			Unwrap(m_PickPixelImage), VK_IMAGE_VIEW_TYPE_2D,
+			VK_FORMAT_R32G32B32A32_SFLOAT,
+			{ VK_CHANNEL_SWIZZLE_R, VK_CHANNEL_SWIZZLE_G, VK_CHANNEL_SWIZZLE_B, VK_CHANNEL_SWIZZLE_A },
+			{ VK_IMAGE_ASPECT_COLOR, 0, 1, 0, 1, },
+			0,
+		};
+
+		// VKTODOMED used for texture display, but eventually will have to be created on the fly
+		// for whichever image we're viewing (and cached), not specifically created here.
+		vkr = vt->CreateImageView(Unwrap(dev), &viewInfo, &m_PickPixelImageView);
+		RDCASSERT(vkr == VK_SUCCESS);
+
+		VKMGR()->WrapResource(Unwrap(dev), m_PickPixelImageView);
+
+		// need to transition image into valid state
+		VkCmdBuffer cmd = driver->GetCmd();
+		VkQueue q = driver->GetQ();
+
+		VkCmdBufferBeginInfo beginInfo = { VK_STRUCTURE_TYPE_CMD_BUFFER_BEGIN_INFO, NULL, VK_CMD_BUFFER_OPTIMIZE_SMALL_BATCH_BIT | VK_CMD_BUFFER_OPTIMIZE_ONE_TIME_SUBMIT_BIT };
+
+		vkr = vt->ResetCommandBuffer(Unwrap(cmd), 0);
+		RDCASSERT(vkr == VK_SUCCESS);
+		vkr = vt->BeginCommandBuffer(Unwrap(cmd), &beginInfo);
+		RDCASSERT(vkr == VK_SUCCESS);
+
+		VkImageMemoryBarrier trans = {
+			VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, NULL,
+			0, 0,
+			VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL,
+			VK_QUEUE_FAMILY_IGNORED, VK_QUEUE_FAMILY_IGNORED,
+			Unwrap(m_PickPixelImage),
+			{ VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1 }
+		};
+
+		void *barrier = (void *)&trans;
+
+		vt->CmdPipelineBarrier(Unwrap(cmd), VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, false, 1, &barrier);
+
+		vt->EndCommandBuffer(Unwrap(cmd));
+
+		vt->QueueSubmit(Unwrap(q), 1, UnwrapPtr(cmd), VK_NULL_HANDLE);
+
+		// VKTODOMED ideally all the commands from Bind to Flip would be recorded
+		// into a single command buffer and we can just have several allocated
+		// ring-buffer style
+		vt->QueueWaitIdle(Unwrap(q));
+
+		// create render pass
+		VkAttachmentDescription attDesc = {
+			VK_STRUCTURE_TYPE_ATTACHMENT_DESCRIPTION, NULL,
+			VK_FORMAT_R32G32B32A32_SFLOAT, 1,
+			VK_ATTACHMENT_LOAD_OP_CLEAR, VK_ATTACHMENT_STORE_OP_STORE,
+			VK_ATTACHMENT_LOAD_OP_DONT_CARE, VK_ATTACHMENT_STORE_OP_DONT_CARE,
+			VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL
+		};
+
+		VkAttachmentReference attRef = { 0, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL };
+
+		VkSubpassDescription sub = {
+			VK_STRUCTURE_TYPE_SUBPASS_DESCRIPTION, NULL,
+			VK_PIPELINE_BIND_POINT_GRAPHICS, 0,
+			0, NULL, // inputs
+			1, &attRef, // color
+			NULL, // resolve
+			{ VK_ATTACHMENT_UNUSED, VK_IMAGE_LAYOUT_UNDEFINED }, // depth-stencil
+			0, NULL, // preserve
+		};
+
+		VkRenderPassCreateInfo rpinfo = {
+				VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO, NULL,
+				1, &attDesc,
+				1, &sub,
+				0, NULL, // dependencies
+		};
+
+		vkr = vt->CreateRenderPass(Unwrap(dev), &rpinfo, &m_PickPixelRP);
+		RDCASSERT(vkr == VK_SUCCESS);
+
+		VKMGR()->WrapResource(Unwrap(dev), m_PickPixelRP);
+
+		// create framebuffer
+		VkFramebufferCreateInfo fbinfo = {
+			VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO, NULL,
+			Unwrap(m_PickPixelRP),
+			1, UnwrapPtr(m_PickPixelImageView),
+			1, 1, 1,
+		};
+
+		vkr = vt->CreateFramebuffer(Unwrap(dev), &fbinfo, &m_PickPixelFB);
+		RDCASSERT(vkr == VK_SUCCESS);
+
+		VKMGR()->WrapResource(Unwrap(dev), m_PickPixelFB);
+
+		m_PickPixelReadbackBuffer.Create(driver, dev, sizeof(float)*4, GPUBuffer::eGPUBufferReadback);
+	}
+
 	VkDescriptorInfo desc[7];
 	RDCEraseEl(desc);
 	
@@ -893,6 +1041,38 @@ VulkanDebugManager::~VulkanDebugManager()
 	m_CheckerboardUBO.Destroy(vt, dev);
 	m_TexDisplayUBO.Destroy(vt, dev);
 
+	m_PickPixelReadbackBuffer.Destroy(vt, dev);
+
+	if(m_PickPixelFB != VK_NULL_HANDLE)
+	{
+		vt->DestroyFramebuffer(Unwrap(dev), Unwrap(m_PickPixelFB));
+		VKMGR()->ReleaseWrappedResource(m_PickPixelFB);
+	}
+
+	if(m_PickPixelRP != VK_NULL_HANDLE)
+	{
+		vt->DestroyRenderPass(Unwrap(dev), Unwrap(m_PickPixelRP));
+		VKMGR()->ReleaseWrappedResource(m_PickPixelRP);
+	}
+
+	if(m_PickPixelImageView != VK_NULL_HANDLE)
+	{
+		vt->DestroyImageView(Unwrap(dev), Unwrap(m_PickPixelImageView));
+		VKMGR()->ReleaseWrappedResource(m_PickPixelImageView);
+	}
+
+	if(m_PickPixelImage != VK_NULL_HANDLE)
+	{
+		vt->DestroyImage(Unwrap(dev), Unwrap(m_PickPixelImage));
+		VKMGR()->ReleaseWrappedResource(m_PickPixelImage);
+	}
+
+	if(m_PickPixelImageMem != VK_NULL_HANDLE)
+	{
+		vt->FreeMemory(Unwrap(dev), Unwrap(m_PickPixelImageMem));
+		VKMGR()->ReleaseWrappedResource(m_PickPixelImageMem);
+	}
+
 	if(m_TextDescSetLayout != VK_NULL_HANDLE)
 	{
 		vt->DestroyDescriptorSetLayout(Unwrap(dev), Unwrap(m_TextDescSetLayout));

renderdoc/driver/vulkan/vk_debug.h

@@ -46,11 +46,15 @@ class VulkanDebugManager
 		~VulkanDebugManager();
 
 		void RenderText(const TextPrintState &textstate, float x, float y, const char *fmt, ...);
-		
+
 		struct GPUBuffer
 		{
+			enum CreateFlags
+			{
+				eGPUBufferReadback = 0x1,
+			};
 			GPUBuffer() : buf(VK_NULL_HANDLE), mem(VK_NULL_HANDLE) {}
-			void Create(WrappedVulkan *driver, VkDevice dev, VkDeviceSize size);
+			void Create(WrappedVulkan *driver, VkDevice dev, VkDeviceSize size, uint32_t flags);
 			void Destroy(const VkLayerDispatchTable *vt, VkDevice dev);
 
 			void FillDescriptor(VkDescriptorInfo &desc);
@@ -85,6 +89,13 @@ class VulkanDebugManager
 		VkDescriptorSet m_TexDisplayDescSet;
 		VkPipeline m_TexDisplayPipeline, m_TexDisplayBlendPipeline;
 		GPUBuffer m_TexDisplayUBO;
+
+		VkDeviceMemory m_PickPixelImageMem;
+		VkImage m_PickPixelImage;
+		VkImageView m_PickPixelImageView;
+		GPUBuffer m_PickPixelReadbackBuffer;
+		VkFramebuffer m_PickPixelFB;
+		VkRenderPass m_PickPixelRP;
 		
 		VkDescriptorSetLayout m_TextDescSetLayout;
 		VkPipelineLayout m_TextPipeLayout;

renderdoc/driver/vulkan/vk_replay.cpp

@@ -426,6 +426,8 @@ VulkanReplay::VulkanReplay()
 	m_OutputWinID = 1;
 	m_ActiveWinID = 0;
 	m_BindDepth = false;
+
+	m_DebugWidth = m_DebugHeight = 1;
 }
 
 VulkanDebugManager *VulkanReplay::GetDebugManager()
@@ -522,18 +524,39 @@ vector<ResourceId> VulkanReplay::GetBuffers()
 
 void VulkanReplay::PickPixel(ResourceId texture, uint32_t x, uint32_t y, uint32_t sliceFace, uint32_t mip, uint32_t sample, float pixel[4])
 {
-	//VULKANNOTIMP("PickPixel");
+	int oldW = m_DebugWidth, oldH = m_DebugHeight;
 
-	ResourceId resid;
-	VkImage fakeBBIm = VK_NULL_HANDLE;
-	VkExtent3D fakeBBext;
-	ResourceFormat fakeBBfmt;
-	m_pDriver->GetFakeBB(resid, fakeBBIm, fakeBBext, fakeBBfmt);
+	m_DebugWidth = m_DebugHeight = 1;
 
-	if(x >= (uint32_t)fakeBBext.width || y >= (uint32_t)fakeBBext.height)
+	// render picked pixel to readback F32 RGBA texture
 	{
-		RDCEraseEl(pixel);
-		return;
+		TextureDisplay texDisplay;
+
+		texDisplay.Red = texDisplay.Green = texDisplay.Blue = texDisplay.Alpha = true;
+		texDisplay.HDRMul = -1.0f;
+		texDisplay.linearDisplayAsGamma = true;
+		texDisplay.FlipY = false;
+		texDisplay.mip = mip;
+		texDisplay.sampleIdx = sample;
+		texDisplay.CustomShader = ResourceId();
+		texDisplay.sliceFace = sliceFace;
+		texDisplay.rangemin = 0.0f;
+		texDisplay.rangemax = 1.0f;
+		texDisplay.scale = 1.0f;
+		texDisplay.texid = texture;
+		texDisplay.rawoutput = true;
+		texDisplay.offx = -float(x);
+		texDisplay.offy = -float(y);
+		
+		VkClearValue clearval = {0};
+		VkRenderPassBeginInfo rpbegin = {
+			VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO, NULL,
+			Unwrap(GetDebugManager()->m_PickPixelRP), Unwrap(GetDebugManager()->m_PickPixelFB),
+			{ { 0, 0, }, { 1, 1 } },
+			1, &clearval,
+		};
+
+		RenderTextureInternal(texDisplay, rpbegin, false);
 	}
 
 	VkDevice dev = m_pDriver->GetDev();
@@ -541,46 +564,21 @@ void VulkanReplay::PickPixel(ResourceId texture, uint32_t x, uint32_t y, uint32_
 	VkQueue q = m_pDriver->GetQ();
 	const VkLayerDispatchTable *vt = ObjDisp(dev);
 
-	// VKTODOMED this should be all created offline, including separate host and
-	// readback buffers
-	// for now while these are very short lived, they are not wrapped 
-	VkDeviceMemory readbackmem = VK_NULL_HANDLE;
-	VkBuffer destbuf = VK_NULL_HANDLE;
+	VkResult vkr = VK_SUCCESS;
 
 	{
-		VkBufferCreateInfo bufInfo = {
-			VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, NULL,
-			128, VK_BUFFER_USAGE_TRANSFER_SOURCE_BIT|VK_BUFFER_USAGE_TRANSFER_DESTINATION_BIT, 0,
-			VK_SHARING_MODE_EXCLUSIVE, 0, NULL,
-		};
-
-		VkResult vkr = vt->CreateBuffer(Unwrap(dev), &bufInfo, &destbuf);
-		RDCASSERT(vkr == VK_SUCCESS);
-		
-		VkMemoryRequirements mrq;
-		vkr = vt->GetBufferMemoryRequirements(Unwrap(dev), destbuf, &mrq);
-		RDCASSERT(vkr == VK_SUCCESS);
-
-		VkMemoryAllocInfo allocInfo = {
-			VK_STRUCTURE_TYPE_MEMORY_ALLOC_INFO, NULL,
-			128, m_pDriver->GetReadbackMemoryIndex(mrq.memoryTypeBits),
-		};
-
-		vkr = vt->AllocMemory(Unwrap(dev), &allocInfo, &readbackmem);
-		RDCASSERT(vkr == VK_SUCCESS);
-
-		vkr = vt->BindBufferMemory(Unwrap(dev), destbuf, readbackmem, 0);
-		RDCASSERT(vkr == VK_SUCCESS);
-
-		// VKTODOHIGH find out the actual current image state
-		VkImageMemoryBarrier fakeTrans = {
+		VkImageMemoryBarrier pickimTrans = {
 			VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, NULL,
-			0, 0, VK_IMAGE_LAYOUT_PRESENT_SOURCE_KHR, VK_IMAGE_LAYOUT_TRANSFER_SOURCE_OPTIMAL,
+			0, 0, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, VK_IMAGE_LAYOUT_TRANSFER_SOURCE_OPTIMAL,
 			VK_QUEUE_FAMILY_IGNORED, VK_QUEUE_FAMILY_IGNORED,
-			Unwrap(fakeBBIm),
+			Unwrap(GetDebugManager()->m_PickPixelImage),
 			{ VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1 }
 		};
 
+		// transition from color attachment to transfer source, with proper memory barriers
+		pickimTrans.outputMask = VK_MEMORY_OUTPUT_COLOR_ATTACHMENT_BIT;
+		pickimTrans.inputMask = VK_MEMORY_INPUT_TRANSFER_BIT;
+
 		VkCmdBufferBeginInfo beginInfo = { VK_STRUCTURE_TYPE_CMD_BUFFER_BEGIN_INFO, NULL, VK_CMD_BUFFER_OPTIMIZE_SMALL_BATCH_BIT | VK_CMD_BUFFER_OPTIMIZE_ONE_TIME_SUBMIT_BIT };
 
 		vkr = vt->ResetCommandBuffer(Unwrap(cmd), 0);
@@ -588,18 +586,23 @@ void VulkanReplay::PickPixel(ResourceId texture, uint32_t x, uint32_t y, uint32_
 		vkr = vt->BeginCommandBuffer(Unwrap(cmd), &beginInfo);
 		RDCASSERT(vkr == VK_SUCCESS);
 
-		void *barrier = (void *)&fakeTrans;
+		void *barrier = (void *)&pickimTrans;
 		vt->CmdPipelineBarrier(Unwrap(cmd), VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, false, 1, &barrier);
-		fakeTrans.oldLayout = fakeTrans.newLayout;
+		pickimTrans.oldLayout = pickimTrans.newLayout;
 
+		pickimTrans.outputMask = 0;
+		pickimTrans.inputMask = 0;
+
+		// do copy
 		VkBufferImageCopy region = {
 			0, 128, 1,
-			{ VK_IMAGE_ASPECT_COLOR, 0, 0}, { (int)x, (int)y, 0 },
+			{ VK_IMAGE_ASPECT_COLOR, 0, 0}, { 0, 0, 0 },
 			{ 1, 1, 1 },
 		};
-		vt->CmdCopyImageToBuffer(Unwrap(cmd), Unwrap(fakeBBIm), VK_IMAGE_LAYOUT_TRANSFER_SOURCE_OPTIMAL, destbuf, 1, &region);
+		vt->CmdCopyImageToBuffer(Unwrap(cmd), Unwrap(GetDebugManager()->m_PickPixelImage), VK_IMAGE_LAYOUT_TRANSFER_SOURCE_OPTIMAL, Unwrap(GetDebugManager()->m_PickPixelReadbackBuffer.buf), 1, &region);
 
-		fakeTrans.newLayout = VK_IMAGE_LAYOUT_PRESENT_SOURCE_KHR;
+		// transition back to color attachment
+		pickimTrans.newLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
 		vt->CmdPipelineBarrier(Unwrap(cmd), VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, false, 1, &barrier);
 
 		vt->EndCommandBuffer(Unwrap(cmd));
@@ -609,34 +612,24 @@ void VulkanReplay::PickPixel(ResourceId texture, uint32_t x, uint32_t y, uint32_
 		vt->QueueWaitIdle(Unwrap(q));
 	}
 
-	byte *pData = NULL;
-	vt->MapMemory(Unwrap(dev), readbackmem, 0, 0, 0, (void **)&pData);
+	float *pData = NULL;
+	vt->MapMemory(Unwrap(dev), Unwrap(GetDebugManager()->m_PickPixelReadbackBuffer.mem), 0, 0, 0, (void **)&pData);
 
 	RDCASSERT(pData != NULL);
 
-	// VKTODOMED this should go through render texture so that we
-	// just get pure F32 RGBA data out
-	if(fakeBBfmt.specialFormat == eSpecial_B8G8R8A8)
+	if(pData == NULL)
 	{
-		pixel[0] = float(pData[2])/255.0f;
-		pixel[1] = float(pData[1])/255.0f;
-		pixel[2] = float(pData[0])/255.0f;
-		pixel[3] = float(pData[3])/255.0f;
+		RDCERR("Failed ot map readback buffer memory");
 	}
 	else
 	{
-		pixel[0] = float(pData[0])/255.0f;
-		pixel[1] = float(pData[1])/255.0f;
-		pixel[2] = float(pData[2])/255.0f;
-		pixel[3] = float(pData[3])/255.0f;
+		pixel[0] = pData[0];
+		pixel[1] = pData[1];
+		pixel[2] = pData[2];
+		pixel[3] = pData[3];
 	}
 
-	vt->UnmapMemory(Unwrap(dev), readbackmem);
-
-	vt->DeviceWaitIdle(Unwrap(dev));
-
-	vt->DestroyBuffer(Unwrap(dev), destbuf);
-	vt->FreeMemory(Unwrap(dev), readbackmem);
+	vt->UnmapMemory(Unwrap(dev), Unwrap(GetDebugManager()->m_PickPixelReadbackBuffer.mem));
 }
 
 uint32_t VulkanReplay::PickVertex(uint32_t frameID, uint32_t eventID, MeshDisplay cfg, uint32_t x, uint32_t y)
@@ -655,7 +648,20 @@ bool VulkanReplay::RenderTexture(TextureDisplay cfg)
 	}
 
 	OutputWindow &outw = it->second;
+	
+	VkClearValue clearval = {0};
+	VkRenderPassBeginInfo rpbegin = {
+		VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO, NULL,
+		Unwrap(outw.renderpass), Unwrap(outw.fb),
+		{ { 0, 0, }, { m_DebugWidth, m_DebugHeight } },
+		1, &clearval,
+	};
 
+	return RenderTextureInternal(cfg, rpbegin, true);
+}
+
+bool VulkanReplay::RenderTextureInternal(TextureDisplay cfg, VkRenderPassBeginInfo rpbegin, bool blendAlpha)
+{
 	VkDevice dev = m_pDriver->GetDev();
 	VkCmdBuffer cmd = m_pDriver->GetCmd();
 	VkQueue q = m_pDriver->GetQ();
@@ -707,20 +713,20 @@ bool VulkanReplay::RenderTexture(TextureDisplay cfg)
 
 	if(cfg.scale <= 0.0f)
 	{
-		float xscale = float(outw.width)/float(tex_x);
-		float yscale = float(outw.height)/float(tex_y);
+		float xscale = float(m_DebugWidth)/float(tex_x);
+		float yscale = float(m_DebugHeight)/float(tex_y);
 
 		float scale = data->Scale = RDCMIN(xscale, yscale);
 
 		if(yscale > xscale)
 		{
 			data->Position.x = 0;
-			data->Position.y = (float(outw.width)-(tex_y*scale) )*0.5f;
+			data->Position.y = (float(m_DebugWidth)-(tex_y*scale) )*0.5f;
 		}
 		else
 		{
 			data->Position.y = 0;
-			data->Position.x = (float(outw.height)-(tex_x*scale) )*0.5f;
+			data->Position.x = (float(m_DebugHeight)-(tex_x*scale) )*0.5f;
 		}
 	}
 
@@ -759,8 +765,8 @@ bool VulkanReplay::RenderTexture(TextureDisplay cfg)
 	data->NumSamples = 1;
 	data->SampleIdx = 0;
 
-	data->OutputRes.x = (float)outw.width;
-	data->OutputRes.y = (float)outw.height;
+	data->OutputRes.x = (float)m_DebugWidth;
+	data->OutputRes.y = (float)m_DebugHeight;
 
 	// VKTODOMED handle different texture types/displays
 	data->OutputDisplayFormat = 0;
@@ -807,20 +813,14 @@ bool VulkanReplay::RenderTexture(TextureDisplay cfg)
 	srcimTrans.oldLayout = srcimTrans.newLayout;
 
 	{
-		VkClearValue clearval = {0};
-		VkRenderPassBeginInfo rpbegin = {
-			VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO, NULL,
-			Unwrap(outw.renderpass), Unwrap(outw.fb),
-			{ { 0, 0, }, { outw.width, outw.height } },
-			1, &clearval,
-		};
 		vt->CmdBeginRenderPass(Unwrap(cmd), &rpbegin, VK_RENDER_PASS_CONTENTS_INLINE);
 
-		// VKTODOMED will need a way to disable blend for other things
-		vt->CmdBindPipeline(Unwrap(cmd), VK_PIPELINE_BIND_POINT_GRAPHICS, cfg.rawoutput ? Unwrap(GetDebugManager()->m_TexDisplayPipeline) : Unwrap(GetDebugManager()->m_TexDisplayBlendPipeline));
+		bool noblend = !cfg.rawoutput || !blendAlpha || cfg.CustomShader != ResourceId();
+
+		vt->CmdBindPipeline(Unwrap(cmd), VK_PIPELINE_BIND_POINT_GRAPHICS, noblend ? Unwrap(GetDebugManager()->m_TexDisplayPipeline) : Unwrap(GetDebugManager()->m_TexDisplayBlendPipeline));
 		vt->CmdBindDescriptorSets(Unwrap(cmd), VK_PIPELINE_BIND_POINT_GRAPHICS, Unwrap(GetDebugManager()->m_TexDisplayPipeLayout), 0, 1, UnwrapPtr(GetDebugManager()->m_TexDisplayDescSet), 0, NULL);
 
-		VkViewport viewport = { 0.0f, 0.0f, (float)outw.width, (float)outw.height, 0.0f, 1.0f };
+		VkViewport viewport = { 0.0f, 0.0f, (float)m_DebugWidth, (float)m_DebugHeight, 0.0f, 1.0f };
 		vt->CmdSetViewport(Unwrap(cmd), 1, &viewport);
 
 		vt->CmdDraw(Unwrap(cmd), 4, 1, 0, 0);
@@ -881,7 +881,7 @@ void VulkanReplay::RenderCheckerboard(Vec3f light, Vec3f dark)
 		VkRenderPassBeginInfo rpbegin = {
 			VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO, NULL,
 			Unwrap(outw.renderpass), Unwrap(outw.fb),
-			{ { 0, 0, }, { outw.width, outw.height } },
+			{ { 0, 0, }, { m_DebugWidth, m_DebugHeight } },
 			1, &clearval,
 		};
 		vt->CmdBeginRenderPass(Unwrap(cmd), &rpbegin, VK_RENDER_PASS_CONTENTS_INLINE);
@@ -889,7 +889,7 @@ void VulkanReplay::RenderCheckerboard(Vec3f light, Vec3f dark)
 		vt->CmdBindPipeline(Unwrap(cmd), VK_PIPELINE_BIND_POINT_GRAPHICS, Unwrap(GetDebugManager()->m_CheckerboardPipeline));
 		vt->CmdBindDescriptorSets(Unwrap(cmd), VK_PIPELINE_BIND_POINT_GRAPHICS, Unwrap(GetDebugManager()->m_CheckerboardPipeLayout), 0, 1, UnwrapPtr(GetDebugManager()->m_CheckerboardDescSet), 0, NULL);
 
-		VkViewport viewport = { 0.0f, 0.0f, (float)outw.width, (float)outw.height, 0.0f, 1.0f };
+		VkViewport viewport = { 0.0f, 0.0f, (float)m_DebugWidth, (float)m_DebugHeight, 0.0f, 1.0f };
 		vt->CmdSetViewport(Unwrap(cmd), 1, &viewport);
 		
 		vt->CmdDraw(Unwrap(cmd), 4, 1, 0, 0);
@@ -947,7 +947,7 @@ void VulkanReplay::RenderHighlightBox(float w, float h, float scale)
 		VkRenderPassBeginInfo rpbegin = {
 			VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO, NULL,
 			Unwrap(outw.renderpass), Unwrap(outw.fb),
-			{ { 0, 0, }, { outw.width, outw.height } },
+			{ { 0, 0, }, { m_DebugWidth, m_DebugHeight } },
 			1, &clearval,
 		};
 		vt->CmdBeginRenderPass(Unwrap(cmd), &rpbegin, VK_RENDER_PASS_CONTENTS_INLINE);
@@ -955,7 +955,7 @@ void VulkanReplay::RenderHighlightBox(float w, float h, float scale)
 		vt->CmdBindPipeline(Unwrap(cmd), VK_PIPELINE_BIND_POINT_GRAPHICS, Unwrap(GetDebugManager()->m_GenericPipeline));
 		vt->CmdBindDescriptorSets(Unwrap(cmd), VK_PIPELINE_BIND_POINT_GRAPHICS, Unwrap(GetDebugManager()->m_GenericPipeLayout), 0, 1, UnwrapPtr(GetDebugManager()->m_GenericDescSet), 0, NULL);
 
-		VkViewport viewport = { 0.0f, 0.0f, (float)outw.width, (float)outw.height, 0.0f, 1.0f };
+		VkViewport viewport = { 0.0f, 0.0f, (float)m_DebugWidth, (float)m_DebugHeight, 0.0f, 1.0f };
 		vt->CmdSetViewport(Unwrap(cmd), 1, &viewport);
 
 		VkDeviceSize zero = 0;
@@ -1041,6 +1041,9 @@ void VulkanReplay::BindOutputWindow(uint64_t id, bool depth)
 
 	OutputWindow &outw = it->second;
 
+	m_DebugWidth = (int32_t)outw.width;
+	m_DebugHeight = (int32_t)outw.height;
+
 	VkDevice dev = m_pDriver->GetDev();
 	VkCmdBuffer cmd = m_pDriver->GetCmd();
 	VkQueue q = m_pDriver->GetQ();

renderdoc/driver/vulkan/vk_replay.h

@@ -209,11 +209,14 @@ class VulkanReplay : public IReplayDriver
 		uint64_t m_OutputWinID;
 		uint64_t m_ActiveWinID;
 		bool m_BindDepth;
+		int m_DebugWidth, m_DebugHeight;
 
 		bool m_Proxy;
 		
 		WrappedVulkan *m_pDriver;
 
+		bool RenderTextureInternal(TextureDisplay cfg, VkRenderPassBeginInfo rpbegin, bool blendAlpha);
+
 		void FillCBufferVariables(rdctype::array<ShaderConstant>, vector<ShaderVariable> &outvars, const vector<byte> &data, size_t &offset);
 
 		VulkanDebugManager *GetDebugManager();