Mitigate environment variables being utterly terrible

* Environment variables are not thread safe on linux, but unfortunately linux is
  obssessed with them so we must hold our nose and interact with them.
* This change tries to move some modifications earlier into init before anything
  else happens to reduce the risk of clashes, as well as removing some reads
  that are unnecessary.

renderdoc/core/core.cpp

@@ -751,6 +751,13 @@ void RenderDoc::Initialise()
   if(IsReplayApp())
     RDCLOGOUTPUT();
 
+  if(IsReplayApp())
+  {
+    for(auto it = m_EnvSetups.begin(); it != m_EnvSetups.end(); ++it)
+      (*it)();
+    m_EnvSetups.clear();
+  }
+
   ProcessConfig();
 }
 

renderdoc/core/core.h

@@ -401,6 +401,7 @@ typedef bool (*VulkanLayerCheck)(VulkanLayerFlags &flags, rdcarray<rdcstr> &myJS
 typedef void (*VulkanLayerInstall)(bool systemLevel);
 
 typedef void (*ShutdownFunction)();
+typedef void (*EnvSetupFunction)();
 
 // this class mediates everything and owns any 'global' resources such as the crash handler.
 //
@@ -536,6 +537,8 @@ public:
   rdcarray<CaptureFileFormat> GetCaptureFileFormats();
   rdcarray<GPUDevice> GetAvailableGPUs();
 
+  void AddEnvSetup(EnvSetupFunction envSetup) { m_EnvSetups.push_back(envSetup); }
+
   void SetVulkanLayerCheck(VulkanLayerCheck callback) { m_VulkanCheck = callback; }
   void SetVulkanLayerInstall(VulkanLayerInstall callback) { m_VulkanInstall = callback; }
   bool NeedVulkanLayerRegistration(VulkanLayerFlags &flags, rdcarray<rdcstr> &myJSONs,
@@ -736,6 +739,7 @@ private:
   VulkanLayerInstall m_VulkanInstall;
 
   rdcarray<ShutdownFunction> m_ShutdownFunctions;
+  rdcarray<EnvSetupFunction> m_EnvSetups;
 
   struct FrameCap
   {

renderdoc/driver/vulkan/vk_replay.cpp

@@ -5640,10 +5640,8 @@ void VulkanReplay::SetProxyBufferData(ResourceId bufid, byte *data, size_t dataS
   VULKANNOTIMP("SetProxyTextureData");
 }
 
-RDResult Vulkan_CreateReplayDevice(RDCFile *rdc, const ReplayOptions &opts, IReplayDriver **driver)
+void VulkanEnvSetup()
 {
-  RDCDEBUG("Creating a VulkanReplay replay device");
-
   // disable the layer env var, just in case the user left it set from a previous capture run
   Process::RegisterEnvironmentModification(
       EnvironmentModification(EnvMod::Set, EnvSep::NoSep, RENDERDOC_VULKAN_LAYER_VAR, "0"));
@@ -5706,6 +5704,11 @@ RDResult Vulkan_CreateReplayDevice(RDCFile *rdc, const ReplayOptions &opts, IRep
       EnvironmentModification(EnvMod::Set, EnvSep::NoSep, "DISABLE_LAYER", "1"));
 
   Process::ApplyEnvironmentModification();
+}
+
+RDResult Vulkan_CreateReplayDevice(RDCFile *rdc, const ReplayOptions &opts, IReplayDriver **driver)
+{
+  RDCDEBUG("Creating a VulkanReplay replay device");
 
   void *module = LoadVulkanLibrary();
 
@@ -5803,6 +5806,7 @@ struct VulkanDriverRegistration
   VulkanDriverRegistration()
   {
     RenderDoc::Inst().RegisterReplayProvider(RDCDriver::Vulkan, &Vulkan_CreateReplayDevice);
+    RenderDoc::Inst().AddEnvSetup(&VulkanEnvSetup);
     RenderDoc::Inst().SetVulkanLayerCheck(&VulkanReplay::CheckVulkanLayer);
     RenderDoc::Inst().SetVulkanLayerInstall(&VulkanReplay::InstallVulkanLayer);
   }

renderdoc/os/posix/posix_process.cpp

@@ -409,9 +409,26 @@ void ApplySingleEnvMod(EnvironmentModification &m, rdcstr &value)
 
 void ApplyEnvironmentModifications(rdcarray<EnvironmentModification> &modifications)
 {
-  // turn environment string to a UTF-8 map
-  char **currentEnvironment = GetCurrentEnvironment();
-  std::map<rdcstr, rdcstr> currentEnv = EnvStringToEnvMap(currentEnvironment);
+  bool needOldEnv = false;
+
+  for(const EnvironmentModification &env : modifications)
+  {
+    if(env.mod != EnvMod::Set)
+    {
+      needOldEnv = true;
+      break;
+    }
+  }
+
+  std::map<rdcstr, rdcstr> currentEnv;
+
+  // turn environment string to a UTF-8 map, but only if we actually need to. Environment variables
+  // are garbage 70s tier tech and so are not thread safe, minimise interaction with them as much as possible
+  if(needOldEnv)
+  {
+    char **currentEnvironment = GetCurrentEnvironment();
+    currentEnv = EnvStringToEnvMap(currentEnvironment);
+  }
 
   for(size_t i = 0; i < modifications.size(); i++)
   {