Change renderdoccmd to be a console program on windows.

* This means it outputs natively/properly to stdout/stderr and its output can be
  redirected with pipes.
* It does mean we need to be very careful whenever it's run internally to not
  pop up a command window, which happens by default.

qrenderdoc/Code/QRDUtils.cpp

@@ -1516,7 +1516,7 @@ bool IsRunningAsAdmin()
 }
 
 bool RunProcessAsAdmin(const QString &fullExecutablePath, const QStringList &params,
-                       QWidget *parent, std::function<void()> finishedCallback)
+                       QWidget *parent, bool hidden, std::function<void()> finishedCallback)
 {
 #if defined(Q_OS_WIN32)
 
@@ -1536,7 +1536,7 @@ bool RunProcessAsAdmin(const QString &fullExecutablePath, const QStringList &par
   info.lpVerb = L"runas";
   info.lpFile = wideExe.c_str();
   info.lpParameters = wideParams.c_str();
-  info.nShow = SW_SHOWNORMAL;
+  info.nShow = hidden ? SW_HIDE : SW_SHOWNORMAL;
 
   ShellExecuteExW(&info);
 

qrenderdoc/Code/QRDUtils.h

@@ -585,7 +585,7 @@ typedef std::function<bool()> ProgressFinishedMethod;
 QStringList ParseArgsList(const QString &args);
 bool IsRunningAsAdmin();
 bool RunProcessAsAdmin(const QString &fullExecutablePath, const QStringList &params,
-                       QWidget *parent = NULL,
+                       QWidget *parent = NULL, bool hidden = false,
                        std::function<void()> finishedCallback = std::function<void()>());
 
 void RevealFilenameInExternalFileBrowser(const QString &filePath);

qrenderdoc/Windows/Dialogs/CaptureDialog.cpp

@@ -431,11 +431,11 @@ void CaptureDialog::vulkanLayerWarn_mouseClick()
         }
 
         RunProcessAsAdmin(cmd, QStringList() << lit("vulkanregister") << lit("--system"), this,
-                          [this]() { ui->vulkanLayerWarn->setVisible(false); });
+                          true, [this]() { ui->vulkanLayerWarn->setVisible(false); });
 #else
         RunProcessAsAdmin(qApp->applicationFilePath(),
                           QStringList() << lit("--install_vulkan_layer") << lit("root"), this,
-                          [this]() { ui->vulkanLayerWarn->setVisible(false); });
+                          false, [this]() { ui->vulkanLayerWarn->setVisible(false); });
 #endif
         return;
       }

qrenderdoc/Windows/Dialogs/UpdateDialog.cpp

@@ -252,9 +252,9 @@ void UpdateDialog::on_update_clicked()
 
       QDir::setCurrent(dir.absolutePath());
 
-      success = RunProcessAsAdmin(dir.absoluteFilePath(cmd), QStringList()
-                                                                 << lit("upgrade") << lit("--path")
-                                                                 << appDir.absolutePath());
+      success = RunProcessAsAdmin(
+          dir.absoluteFilePath(cmd),
+          QStringList() << lit("upgrade") << lit("--path") << appDir.absolutePath(), NULL, true);
 
       exit(0);
 

qrenderdoc/Windows/MainWindow.cpp

@@ -1019,6 +1019,9 @@ void MainWindow::SetTitle(const QString &filename)
                 .arg(lit(FULL_VERSION_STRING))
                 .arg(QString::fromLatin1(GitVersionHash));
 
+  if(IsRunningAsAdmin())
+    text += tr(" (Administrator)");
+
   if(QString::fromLatin1(RENDERDOC_GetVersionString()) != lit(MAJOR_MINOR_VERSION_STRING))
     text += tr(" - !! VERSION MISMATCH DETECTED !!");
 

renderdoc/core/crash_handler.h

@@ -124,6 +124,11 @@ public:
     RDCEraseEl(pi);
     RDCEraseEl(si);
 
+    // hide the console window
+    si.cb = sizeof(si);
+    si.dwFlags |= STARTF_USESHOWWINDOW;
+    si.wShowWindow = SW_HIDE;
+
     HANDLE waitEvent = CreateEventA(NULL, TRUE, FALSE, "RENDERDOC_CRASHHANDLE");
 
     wchar_t radpath[MAX_PATH] = {0};
@@ -157,7 +162,8 @@ public:
 
     wcscpy_s(paramsAlloc, 511, cmdline.c_str());
 
-    BOOL ret = CreateProcessW(NULL, paramsAlloc, NULL, NULL, FALSE, 0, NULL, NULL, &si, &pi);
+    BOOL ret = CreateProcessW(NULL, paramsAlloc, NULL, NULL, FALSE, CREATE_NEW_CONSOLE, NULL, NULL,
+                              &si, &pi);
 
     if(!ret)
       RDCERR("Failed to create crashhandle server: %d", GetLastError());

renderdoc/os/win32/win32_process.cpp

@@ -781,6 +781,11 @@ ExecuteResult Process::InjectIntoProcess(uint32_t pid, const rdcarray<Environmen
     RDCEraseEl(pSec);
     RDCEraseEl(tSec);
 
+    // hide the console window
+    si.cb = sizeof(si);
+    si.dwFlags |= STARTF_USESHOWWINDOW;
+    si.wShowWindow = SW_HIDE;
+
     pSec.nLength = sizeof(pSec);
     tSec.nLength = sizeof(tSec);
 
@@ -863,8 +868,8 @@ ExecuteResult Process::InjectIntoProcess(uint32_t pid, const rdcarray<Environmen
       commandLine = (wchar_t *)cmdWithEnv.c_str();
     }
 
-    BOOL retValue = CreateProcessW(NULL, commandLine, &pSec, &tSec, false, CREATE_SUSPENDED, NULL,
-                                   NULL, &si, &pi);
+    BOOL retValue = CreateProcessW(NULL, commandLine, &pSec, &tSec, false,
+                                   CREATE_NEW_CONSOLE | CREATE_SUSPENDED, NULL, NULL, &si, &pi);
 
     SAFE_DELETE_ARRAY(paramsAlloc);
 
@@ -1440,6 +1445,8 @@ bool Process::StartGlobalHook(const char *pathmatch, const char *capturefile,
   pSec.nLength = sizeof(pSec);
   tSec.nLength = sizeof(tSec);
 
+  si.cb = sizeof(si);
+
   std::wstring paramsAlloc;
   paramsAlloc.resize(2048);
 
@@ -1461,7 +1468,10 @@ bool Process::StartGlobalHook(const char *pathmatch, const char *capturefile,
   paramsAlloc[2047] = 0;
 
   // we'll be setting stdin
-  si.dwFlags |= STARTF_USESTDHANDLES;
+  si.dwFlags = STARTF_USESHOWWINDOW | STARTF_USESTDHANDLES;
+
+  // hide the console window
+  si.wShowWindow = SW_HIDE;
 
   // this is the end of the pipe that the child will inherit and use as stdin
   HANDLE childEnd = NULL;
@@ -1497,7 +1507,8 @@ bool Process::StartGlobalHook(const char *pathmatch, const char *capturefile,
   }
 
   // launch the process
-  BOOL retValue = CreateProcessW(NULL, &paramsAlloc[0], &pSec, &tSec, true, 0, NULL, NULL, &si, &pi);
+  BOOL retValue = CreateProcessW(NULL, &paramsAlloc[0], &pSec, &tSec, true, CREATE_NEW_CONSOLE,
+                                 NULL, NULL, &si, &pi);
 
   // we don't need this end anymore, the child has it
   CloseHandle(childEnd);
@@ -1553,7 +1564,8 @@ bool Process::StartGlobalHook(const char *pathmatch, const char *capturefile,
     si.hStdInput = childEnd;
   }
 
-  retValue = CreateProcessW(NULL, &paramsAlloc[0], &pSec, &tSec, true, 0, NULL, NULL, &si, &pi);
+  retValue = CreateProcessW(NULL, &paramsAlloc[0], &pSec, &tSec, true, CREATE_NEW_CONSOLE, NULL,
+                            NULL, &si, &pi);
 
   // we don't need this end anymore
   CloseHandle(childEnd);

renderdoc/os/win32/win32_stringio.cpp

@@ -869,9 +869,9 @@ void WriteOutput(int channel, const char *str)
   if(channel == OSUtility::Output_DebugMon)
     OutputDebugStringW(wstr.c_str());
   else if(channel == OSUtility::Output_StdOut)
-    WriteConsoleW(GetStdHandle(STD_OUTPUT_HANDLE), wstr.c_str(), DWORD(wstr.size()), NULL, NULL);
+    fwprintf_s(stdout, L"%s", wstr.c_str());
   else if(channel == OSUtility::Output_StdErr)
-    WriteConsoleW(GetStdHandle(STD_ERROR_HANDLE), wstr.c_str(), DWORD(wstr.size()), NULL, NULL);
+    fwprintf_s(stderr, L"%s", wstr.c_str());
 }
 
 uint64_t GetMachineIdent()

renderdoccmd/renderdoccmd.vcxproj

@@ -107,7 +107,7 @@
       <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
     </ClCompile>
     <Link>
-      <SubSystem>Windows</SubSystem>
+      <SubSystem>Console</SubSystem>
       <GenerateDebugInformation>true</GenerateDebugInformation>
       <AdditionalDependencies>psapi.lib;ws2_32.lib;Wininet.lib;%(AdditionalDependencies)</AdditionalDependencies>
     </Link>
@@ -128,7 +128,7 @@
       <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
     </ClCompile>
     <Link>
-      <SubSystem>Windows</SubSystem>
+      <SubSystem>Console</SubSystem>
       <GenerateDebugInformation>true</GenerateDebugInformation>
       <AdditionalDependencies>psapi.lib;ws2_32.lib;Wininet.lib;%(AdditionalDependencies)</AdditionalDependencies>
     </Link>
@@ -149,7 +149,7 @@
       <TreatWarningAsError>true</TreatWarningAsError>
     </ClCompile>
     <Link>
-      <SubSystem>Windows</SubSystem>
+      <SubSystem>Console</SubSystem>
       <GenerateDebugInformation>true</GenerateDebugInformation>
       <EnableCOMDATFolding>true</EnableCOMDATFolding>
       <OptimizeReferences>true</OptimizeReferences>
@@ -172,7 +172,7 @@
       <TreatWarningAsError>true</TreatWarningAsError>
     </ClCompile>
     <Link>
-      <SubSystem>Windows</SubSystem>
+      <SubSystem>Console</SubSystem>
       <GenerateDebugInformation>true</GenerateDebugInformation>
       <EnableCOMDATFolding>true</EnableCOMDATFolding>
       <OptimizeReferences>true</OptimizeReferences>

renderdoccmd/renderdoccmd_win32.cpp

@@ -752,21 +752,12 @@ std::string getParentExe()
   return "";
 }
 
-int WINAPI wWinMain(_In_ HINSTANCE hInst, _In_opt_ HINSTANCE hPrevInstance, _In_ LPWSTR lpCmdLine,
-                    _In_ int nShowCmd)
+// ignore the argc/argv we get here, convert from wide to be sure we're unicode safe.
+int main(int, char *)
 {
   LPWSTR *wargv;
   int argc;
 
-  if(AttachConsole(ATTACH_PARENT_PROCESS))
-  {
-    freopen("CONOUT$", "w", stdout);
-    freopen("CONOUT$", "w", stderr);
-
-    std::cout.sync_with_stdio();
-    std::cerr.sync_with_stdio();
-  }
-
   wargv = CommandLineToArgvW(GetCommandLine(), &argc);
 
   std::vector<std::string> argv;
@@ -789,13 +780,7 @@ int WINAPI wWinMain(_In_ HINSTANCE hInst, _In_opt_ HINSTANCE hPrevInstance, _In_
       parent[i] = '/';
   }
 
-  if(strstr(parent.c_str(), "/cmd.exe") && AttachConsole(ATTACH_PARENT_PROCESS))
-  {
-    freopen("CONOUT$", "w", stdout);
-    freopen("CONOUT$", "w", stderr);
-  }
-
-  hInstance = hInst;
+  hInstance = GetModuleHandleA(NULL);
 
   WNDCLASSEX wc;
   wc.cbSize = sizeof(WNDCLASSEX);