diff --git a/renderdocui/Code/AppMain.cs b/renderdocui/Code/AppMain.cs index 191956864..898dd8b02 100644 --- a/renderdocui/Code/AppMain.cs +++ b/renderdocui/Code/AppMain.cs @@ -60,6 +60,12 @@ namespace renderdocui.Code return; } + if (args.Contains("--registerVKLayer")) + { + Helpers.RegisterVulkanLayer(); + return; + } + Win32PInvoke.LoadLibrary("renderdoc.dll"); // clean up any update that just happened @@ -155,6 +161,19 @@ namespace renderdocui.Code { cfg.CheckUpdate_UpdateAvailable = false; cfg.CheckUpdate_UpdateResponse = ""; + + bool hasOtherJSON; + bool thisRegistered; + string[] otherJSONs; + + bool configured = Helpers.CheckVulkanLayerRegistration(out hasOtherJSON, out thisRegistered, out otherJSONs); + + // if nothing is configured (ie. no other JSON files), then set up our layer + // as part of the update process. + if (!configured && !hasOtherJSON && !thisRegistered) + { + Helpers.RegisterVulkanLayer(); + } } } diff --git a/renderdocui/Code/Helpers.cs b/renderdocui/Code/Helpers.cs index cfc5c10c2..784d5d981 100644 --- a/renderdocui/Code/Helpers.cs +++ b/renderdocui/Code/Helpers.cs @@ -185,6 +185,166 @@ namespace renderdocui.Code RefreshAssociations(); } + + private static string GetVulkanJSONPath(bool wow6432) + { + string basepath = Path.GetDirectoryName(Application.ExecutablePath); + if (wow6432) + basepath = Path.Combine(basepath, "x86"); + + return Path.Combine(basepath, "renderdoc.json"); + } + + private static RegistryKey GetVulkanImplicitLayersKey(bool write, bool wow6432) + { + try + { + string basepath = "SOFTWARE\\"; + if (wow6432) + basepath += "Wow6432Node\\"; + + if(write) + return Registry.LocalMachine.CreateSubKey(basepath + "Khronos\\Vulkan\\ImplicitLayers"); + else + return Registry.LocalMachine.OpenSubKey(basepath + "Khronos\\Vulkan\\ImplicitLayers"); + } + catch (Exception) + { + } + return null; + } + + public static bool CheckVulkanLayerRegistration(out bool hasOtherJSON, out bool thisRegistered, out string[] otherJSONs) + { + RegistryKey key = GetVulkanImplicitLayersKey(false, false); + + // if we couldn't even get the ImplicitLayers reg key the system doesn't have the + // vulkan runtime, so we return as if we are not registered (as that's the case). + // People not using vulkan can either ignore the message, or click to set it up + // and it will go away as we'll have rights to create it. + if (key == null) + { + hasOtherJSON = false; + otherJSONs = new string[] { }; + thisRegistered = false; + return false; + } + + string myJSON = Path.GetFullPath(GetVulkanJSONPath(false)); + + string[] names = key.GetValueNames(); + + // defaults + thisRegistered = false; + hasOtherJSON = false; + + List others = new List(); + + foreach (var n in names) + { + if(Path.GetFullPath(n) == myJSON) + { + thisRegistered = true; + } + else if(n.IndexOf("renderdoc.json") > 0) + { + hasOtherJSON = true; + others.Add(Path.GetFullPath(n)); + } + } + + // if we're 64-bit update that too. For 32-bit the above path covers it. + if (Environment.Is64BitProcess) + { + myJSON = Path.GetFullPath(GetVulkanJSONPath(true)); + key = GetVulkanImplicitLayersKey(false, true); + + names = key.GetValueNames(); + + foreach (var n in names) + { + if (Path.GetFullPath(n) == myJSON) + { + thisRegistered = true; + } + else if (n.IndexOf("renderdoc.json") > 0) + { + hasOtherJSON = true; + others.Add(Path.GetFullPath(n)); + } + } + } + + if(hasOtherJSON) + otherJSONs = others.ToArray(); + else + otherJSONs = new string[] {}; + + // return true if all is OK + return !hasOtherJSON && thisRegistered; + } + + public static bool CheckVulkanLayerRegistration() + { + bool dummy1, dummy2; + string[] dummy3; + return CheckVulkanLayerRegistration(out dummy1, out dummy2, out dummy3); + } + + public static void RegisterVulkanLayer() + { + if (!IsElevated) + { + var process = new Process(); + process.StartInfo = new ProcessStartInfo(Application.ExecutablePath, "--registerVKLayer"); + process.StartInfo.Verb = "runas"; + try + { + process.Start(); + // wait for process to finish + process.WaitForExit(); + } + catch (Exception) + { + } + return; + } + + // we know we're elevated, so open the key for write + RegistryKey key = GetVulkanImplicitLayersKey(true, false); + + if (key != null) + { + string[] names = key.GetValueNames(); + + // for simplicity we just delete *all* renderdoc.json values, then + // add our own, even if it was there before. + foreach (var n in names) + if (n.IndexOf("renderdoc.json") > 0) + key.DeleteValue(n); + + key.SetValue(GetVulkanJSONPath(false), (uint)0, RegistryValueKind.DWord); + } + + // if we're 64-bit update that too. For 32-bit the above path covers it. + if (Environment.Is64BitProcess) + { + key = GetVulkanImplicitLayersKey(true, true); + + if (key != null) + { + string[] names = key.GetValueNames(); + + // for simplicity we just delete *all* renderdoc.json values, then + // add our own, even if it was there before. + foreach (var n in names) + if (n.IndexOf("renderdoc.json") > 0) + key.DeleteValue(n); + + key.SetValue(GetVulkanJSONPath(true), (uint)0, RegistryValueKind.DWord); + } + } + } } // KeyValuePair isn't serializable, so we make our own that is diff --git a/renderdocui/Windows/Dialogs/CaptureDialog.Designer.cs b/renderdocui/Windows/Dialogs/CaptureDialog.Designer.cs index bc3e525cb..64dbd818e 100644 --- a/renderdocui/Windows/Dialogs/CaptureDialog.Designer.cs +++ b/renderdocui/Windows/Dialogs/CaptureDialog.Designer.cs @@ -79,6 +79,7 @@ this.tableLayoutPanel2 = new System.Windows.Forms.TableLayoutPanel(); this.programGroup = new System.Windows.Forms.GroupBox(); this.processGroup = new System.Windows.Forms.GroupBox(); + this.vulkanLayerWarn = new System.Windows.Forms.Button(); label2 = new System.Windows.Forms.Label(); label1 = new System.Windows.Forms.Label(); pid = ((System.Windows.Forms.ColumnHeader)(new System.Windows.Forms.ColumnHeader())); @@ -134,7 +135,7 @@ | System.Windows.Forms.AnchorStyles.Right))); label5.Location = new System.Drawing.Point(9, 16); label5.Name = "label5"; - label5.Size = new System.Drawing.Size(473, 23); + label5.Size = new System.Drawing.Size(475, 23); label5.TabIndex = 3; label5.Text = "NOTE: Injecting only works when the process has not used the target API"; // @@ -157,10 +158,10 @@ this.actionsGroup.AutoSize = true; this.actionsGroup.AutoSizeMode = System.Windows.Forms.AutoSizeMode.GrowAndShrink; this.actionsGroup.Controls.Add(this.actionsFlow); - this.actionsGroup.Location = new System.Drawing.Point(10, 500); + this.actionsGroup.Location = new System.Drawing.Point(10, 558); this.actionsGroup.Margin = new System.Windows.Forms.Padding(10); this.actionsGroup.Name = "actionsGroup"; - this.actionsGroup.Size = new System.Drawing.Size(195, 65); + this.actionsGroup.Size = new System.Drawing.Size(197, 65); this.actionsGroup.TabIndex = 11; this.actionsGroup.TabStop = false; this.actionsGroup.Text = "Actions"; @@ -217,10 +218,10 @@ this.globalGroup.AutoSize = true; this.globalGroup.AutoSizeMode = System.Windows.Forms.AutoSizeMode.GrowAndShrink; this.globalGroup.Controls.Add(this.globalFlow); - this.globalGroup.Location = new System.Drawing.Point(10, 585); + this.globalGroup.Location = new System.Drawing.Point(10, 643); this.globalGroup.Margin = new System.Windows.Forms.Padding(10); this.globalGroup.Name = "globalGroup"; - this.globalGroup.Size = new System.Drawing.Size(195, 80); + this.globalGroup.Size = new System.Drawing.Size(197, 80); this.globalGroup.TabIndex = 12; this.globalGroup.TabStop = false; this.globalGroup.Text = "Global Process Hook"; @@ -256,10 +257,10 @@ this.capOptsGroup.AutoSize = true; this.capOptsGroup.AutoSizeMode = System.Windows.Forms.AutoSizeMode.GrowAndShrink; this.capOptsGroup.Controls.Add(this.capOptsFlow); - this.capOptsGroup.Location = new System.Drawing.Point(10, 367); + this.capOptsGroup.Location = new System.Drawing.Point(10, 425); this.capOptsGroup.Margin = new System.Windows.Forms.Padding(10); this.capOptsGroup.Name = "capOptsGroup"; - this.capOptsGroup.Size = new System.Drawing.Size(195, 113); + this.capOptsGroup.Size = new System.Drawing.Size(197, 113); this.capOptsGroup.TabIndex = 4; this.capOptsGroup.TabStop = false; this.capOptsGroup.Text = "Capture Options"; @@ -421,7 +422,6 @@ this.VerifyMapWrites.Size = new System.Drawing.Size(130, 20); this.VerifyMapWrites.TabIndex = 18; this.VerifyMapWrites.Text = "Verify Map() Writes"; - this.toolTip.SetToolTip(this.AutoStart, "When enabled, causes Map() pointers to be validated to check for buffer overruns."); this.VerifyMapWrites.UseVisualStyleBackColor = true; // // AutoStart @@ -443,9 +443,9 @@ this.panel2.Controls.Add(this.save); this.panel2.Controls.Add(this.close); this.panel2.Controls.Add(this.capture); - this.panel2.Location = new System.Drawing.Point(3, 678); + this.panel2.Location = new System.Drawing.Point(3, 736); this.panel2.Name = "panel2"; - this.panel2.Size = new System.Drawing.Size(209, 26); + this.panel2.Size = new System.Drawing.Size(211, 26); this.panel2.TabIndex = 8; // // load @@ -475,7 +475,7 @@ // close // this.close.Anchor = ((System.Windows.Forms.AnchorStyles)((System.Windows.Forms.AnchorStyles.Bottom | System.Windows.Forms.AnchorStyles.Right))); - this.close.Location = new System.Drawing.Point(162, 3); + this.close.Location = new System.Drawing.Point(164, 3); this.close.Margin = new System.Windows.Forms.Padding(0); this.close.Name = "close"; this.close.Size = new System.Drawing.Size(41, 23); @@ -487,7 +487,7 @@ // capture // this.capture.Anchor = ((System.Windows.Forms.AnchorStyles)((System.Windows.Forms.AnchorStyles.Bottom | System.Windows.Forms.AnchorStyles.Right))); - this.capture.Location = new System.Drawing.Point(105, 3); + this.capture.Location = new System.Drawing.Point(107, 3); this.capture.Margin = new System.Windows.Forms.Padding(0); this.capture.Name = "capture"; this.capture.Size = new System.Drawing.Size(52, 23); @@ -538,14 +538,14 @@ | System.Windows.Forms.AnchorStyles.Right))); this.cmdline.Location = new System.Drawing.Point(137, 71); this.cmdline.Name = "cmdline"; - this.cmdline.Size = new System.Drawing.Size(52, 20); + this.cmdline.Size = new System.Drawing.Size(54, 20); this.cmdline.TabIndex = 4; this.toolTip.SetToolTip(this.cmdline, "The command-line that will be passed to the executable on launch"); // // workDirBrowse // this.workDirBrowse.Anchor = ((System.Windows.Forms.AnchorStyles)((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Right))); - this.workDirBrowse.Location = new System.Drawing.Point(165, 45); + this.workDirBrowse.Location = new System.Drawing.Point(167, 45); this.workDirBrowse.Name = "workDirBrowse"; this.workDirBrowse.Size = new System.Drawing.Size(24, 20); this.workDirBrowse.TabIndex = 3; @@ -560,7 +560,7 @@ | System.Windows.Forms.AnchorStyles.Right))); this.workDirPath.Location = new System.Drawing.Point(137, 45); this.workDirPath.Name = "workDirPath"; - this.workDirPath.Size = new System.Drawing.Size(22, 20); + this.workDirPath.Size = new System.Drawing.Size(24, 20); this.workDirPath.TabIndex = 2; this.toolTip.SetToolTip(this.workDirPath, "The working directory the executable will be launched in"); this.workDirPath.TextChanged += new System.EventHandler(this.workDirPath_TextChanged); @@ -570,7 +570,7 @@ // exeBrowse // this.exeBrowse.Anchor = ((System.Windows.Forms.AnchorStyles)((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Right))); - this.exeBrowse.Location = new System.Drawing.Point(165, 18); + this.exeBrowse.Location = new System.Drawing.Point(167, 18); this.exeBrowse.Name = "exeBrowse"; this.exeBrowse.Size = new System.Drawing.Size(24, 20); this.exeBrowse.TabIndex = 1; @@ -586,7 +586,7 @@ | System.Windows.Forms.AnchorStyles.Right))); this.exePath.Location = new System.Drawing.Point(137, 19); this.exePath.Name = "exePath"; - this.exePath.Size = new System.Drawing.Size(22, 20); + this.exePath.Size = new System.Drawing.Size(24, 20); this.exePath.TabIndex = 0; this.toolTip.SetToolTip(this.exePath, "The executable file to launch"); this.exePath.TextChanged += new System.EventHandler(this.exePath_TextChanged); @@ -596,7 +596,7 @@ // pidRefresh // this.pidRefresh.Anchor = ((System.Windows.Forms.AnchorStyles)((System.Windows.Forms.AnchorStyles.Bottom | System.Windows.Forms.AnchorStyles.Right))); - this.pidRefresh.Location = new System.Drawing.Point(410, 180); + this.pidRefresh.Location = new System.Drawing.Point(412, 180); this.pidRefresh.Name = "pidRefresh"; this.pidRefresh.Size = new System.Drawing.Size(75, 23); this.pidRefresh.TabIndex = 6; @@ -619,7 +619,7 @@ this.pidList.Location = new System.Drawing.Point(6, 42); this.pidList.MultiSelect = false; this.pidList.Name = "pidList"; - this.pidList.Size = new System.Drawing.Size(183, 129); + this.pidList.Size = new System.Drawing.Size(185, 129); this.pidList.TabIndex = 5; this.toolTip.SetToolTip(this.pidList, "Select the process to inject into - must not yet have utilised the target API"); this.pidList.UseCompatibleStateImageBehavior = false; @@ -634,21 +634,23 @@ this.tableLayoutPanel2.ColumnCount = 1; this.tableLayoutPanel2.ColumnStyles.Add(new System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Percent, 100F)); this.tableLayoutPanel2.Controls.Add(this.programGroup, 0, 0); - this.tableLayoutPanel2.Controls.Add(this.panel2, 0, 5); - this.tableLayoutPanel2.Controls.Add(this.capOptsGroup, 0, 2); + this.tableLayoutPanel2.Controls.Add(this.panel2, 0, 6); + this.tableLayoutPanel2.Controls.Add(this.capOptsGroup, 0, 3); this.tableLayoutPanel2.Controls.Add(this.processGroup, 0, 1); - this.tableLayoutPanel2.Controls.Add(this.actionsGroup, 0, 3); - this.tableLayoutPanel2.Controls.Add(this.globalGroup, 0, 4); + this.tableLayoutPanel2.Controls.Add(this.actionsGroup, 0, 4); + this.tableLayoutPanel2.Controls.Add(this.globalGroup, 0, 5); + this.tableLayoutPanel2.Controls.Add(this.vulkanLayerWarn, 0, 2); this.tableLayoutPanel2.Location = new System.Drawing.Point(0, 0); this.tableLayoutPanel2.Name = "tableLayoutPanel2"; - this.tableLayoutPanel2.RowCount = 6; + this.tableLayoutPanel2.RowCount = 7; this.tableLayoutPanel2.RowStyles.Add(new System.Windows.Forms.RowStyle()); this.tableLayoutPanel2.RowStyles.Add(new System.Windows.Forms.RowStyle()); this.tableLayoutPanel2.RowStyles.Add(new System.Windows.Forms.RowStyle()); this.tableLayoutPanel2.RowStyles.Add(new System.Windows.Forms.RowStyle()); this.tableLayoutPanel2.RowStyles.Add(new System.Windows.Forms.RowStyle()); this.tableLayoutPanel2.RowStyles.Add(new System.Windows.Forms.RowStyle()); - this.tableLayoutPanel2.Size = new System.Drawing.Size(215, 707); + this.tableLayoutPanel2.RowStyles.Add(new System.Windows.Forms.RowStyle()); + this.tableLayoutPanel2.Size = new System.Drawing.Size(217, 765); this.tableLayoutPanel2.TabIndex = 8; // // programGroup @@ -666,7 +668,7 @@ this.programGroup.Location = new System.Drawing.Point(10, 10); this.programGroup.Margin = new System.Windows.Forms.Padding(10); this.programGroup.Name = "programGroup"; - this.programGroup.Size = new System.Drawing.Size(195, 108); + this.programGroup.Size = new System.Drawing.Size(197, 108); this.programGroup.TabIndex = 10; this.programGroup.TabStop = false; this.programGroup.Text = "Program"; @@ -681,18 +683,38 @@ this.processGroup.Location = new System.Drawing.Point(10, 138); this.processGroup.Margin = new System.Windows.Forms.Padding(10); this.processGroup.Name = "processGroup"; - this.processGroup.Size = new System.Drawing.Size(195, 209); + this.processGroup.Size = new System.Drawing.Size(197, 209); this.processGroup.TabIndex = 9; this.processGroup.TabStop = false; this.processGroup.Text = "Process"; // + // vulkanLayerWarn + // + this.vulkanLayerWarn.Anchor = ((System.Windows.Forms.AnchorStyles)(((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Left) + | System.Windows.Forms.AnchorStyles.Right))); + this.vulkanLayerWarn.BackColor = System.Drawing.SystemColors.Info; + this.vulkanLayerWarn.Cursor = System.Windows.Forms.Cursors.Hand; + this.vulkanLayerWarn.FlatStyle = System.Windows.Forms.FlatStyle.Flat; + this.vulkanLayerWarn.Image = global::renderdocui.Properties.Resources.information; + this.vulkanLayerWarn.ImageAlign = System.Drawing.ContentAlignment.MiddleLeft; + this.vulkanLayerWarn.Location = new System.Drawing.Point(3, 367); + this.vulkanLayerWarn.Margin = new System.Windows.Forms.Padding(3, 10, 3, 10); + this.vulkanLayerWarn.Name = "vulkanLayerWarn"; + this.vulkanLayerWarn.Size = new System.Drawing.Size(211, 38); + this.vulkanLayerWarn.TabIndex = 13; + this.vulkanLayerWarn.Text = "Warning: Vulkan capture is not configured. Click here to set up Vulkan capture."; + this.vulkanLayerWarn.TextAlign = System.Drawing.ContentAlignment.MiddleLeft; + this.vulkanLayerWarn.TextImageRelation = System.Windows.Forms.TextImageRelation.ImageBeforeText; + this.vulkanLayerWarn.UseVisualStyleBackColor = false; + this.vulkanLayerWarn.Click += new System.EventHandler(this.vulkanLayerWarn_Click); + // // CaptureDialog // this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F); this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; this.AutoScroll = true; this.AutoScrollMinSize = new System.Drawing.Size(215, 0); - this.ClientSize = new System.Drawing.Size(215, 736); + this.ClientSize = new System.Drawing.Size(214, 738); this.Controls.Add(this.tableLayoutPanel2); this.Font = new System.Drawing.Font("Microsoft Sans Serif", 8.25F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0))); this.Name = "CaptureDialog"; @@ -770,6 +792,7 @@ private System.Windows.Forms.GroupBox actionsGroup; private System.Windows.Forms.GroupBox globalGroup; private System.Windows.Forms.Label globalLabel; + private System.Windows.Forms.Button vulkanLayerWarn; } } \ No newline at end of file diff --git a/renderdocui/Windows/Dialogs/CaptureDialog.cs b/renderdocui/Windows/Dialogs/CaptureDialog.cs index 5c61f8b70..8f8a685ed 100644 --- a/renderdocui/Windows/Dialogs/CaptureDialog.cs +++ b/renderdocui/Windows/Dialogs/CaptureDialog.cs @@ -174,6 +174,8 @@ namespace renderdocui.Windows.Dialogs Icon = global::renderdocui.Properties.Resources.icon; + vulkanLayerWarn.Visible = !Helpers.CheckVulkanLayerRegistration(); + var defaults = new CaptureSettings(); defaults.Inject = false; @@ -885,5 +887,65 @@ namespace renderdocui.Windows.Dialogs if (toggleGlobalHook.Checked) toggleGlobalHook.Checked = false; } + + private void vulkanLayerWarn_Click(object sender, EventArgs e) + { + string caption = "Configure Vulkan layer settings in registry?"; + + bool hasOtherJSON = false; + bool thisRegistered = false; + string[] otherJSONs = new string[] {}; + + Helpers.CheckVulkanLayerRegistration(out hasOtherJSON, out thisRegistered, out otherJSONs); + + string myJSON = Path.GetDirectoryName(Application.ExecutablePath) + Path.DirectorySeparatorChar + "renderdoc.json"; + + string msg = "Vulkan capture happens through the API's layer mechanism. RenderDoc has detected that "; + + if (hasOtherJSON) + { + msg += "there " + (otherJSONs.Length > 1 ? "are other RenderDoc builds" : "is another RenderDoc build") + + " registered already. " + (otherJSONs.Length > 1 ? "They" : "It") + + " must be disabled so that capture can happen without nasty clashes."; + + if (!thisRegistered) + msg += " Also "; + } + + if (!thisRegistered) + { + msg += "the layer for this installation is not yet registered. This could be due to an " + + "upgrade from a version that didn't support Vulkan, or if this version is just a loose unzip/dev build."; + } + + msg += "\n\nWould you like to proceed with the following changes?\n\n"; + + if (hasOtherJSON) + { + foreach (var j in otherJSONs) + msg += "Unregister: " + j + "\n"; + + msg += "\n"; + } + + if (!thisRegistered) + { + msg += "Register: " + myJSON + "\n"; + if (Environment.Is64BitProcess) + msg += "Register: " + myJSON.Replace("renderdoc.json", @"x86\renderdoc.json") + "\n"; + msg += "\n"; + } + + msg += "This is a one-off change to the registry, it won't be needed again unless the installation moves."; + + DialogResult res = MessageBox.Show(msg, caption, MessageBoxButtons.YesNoCancel, MessageBoxIcon.Question); + + if (res == DialogResult.Yes) + { + Helpers.RegisterVulkanLayer(); + + vulkanLayerWarn.Visible = !Helpers.CheckVulkanLayerRegistration(); + } + } } } diff --git a/renderdocui/Windows/Dialogs/CaptureDialog.resx b/renderdocui/Windows/Dialogs/CaptureDialog.resx index 8e85281e7..54cac7c45 100644 --- a/renderdocui/Windows/Dialogs/CaptureDialog.resx +++ b/renderdocui/Windows/Dialogs/CaptureDialog.resx @@ -147,7 +147,4 @@ 382, 17 - - 485, 17 - \ No newline at end of file