From e4b0d5fbae72a48705d578a72eb5cce8c3f11bc6 Mon Sep 17 00:00:00 2001 From: cdozdil Date: Fri, 13 Nov 2026 10:20:54 +0300 Subject: [PATCH] Don't take the loader lock while holding a Streamline plugin mutex LoadLibraryCheckW waits for a plugin mutex after the load, also on threads that already hold the loader lock (nested loads), while hookPlugin, releaseDeadSlots and the CheckModulesInMemory census called GetProcAddress, GetModuleFileNameW and GetModuleHandleW with the mutex held, which can take the loader lock (always under Wine). slGetPluginFunction is now found by reading the export table, slots are checked by their mapped file name, and the census looks up modules before taking the try_lock. --- OptiScaler/hooks/LibraryLoad_Hooks.cpp | 76 ++++++++++++++------------ OptiScaler/hooks/Streamline_Hooks.cpp | 43 +++++++++++++-- OptiScaler/hooks/Streamline_Hooks.h | 4 +- 3 files changed, 82 insertions(+), 41 deletions(-) diff --git a/OptiScaler/hooks/LibraryLoad_Hooks.cpp b/OptiScaler/hooks/LibraryLoad_Hooks.cpp index e30b05ae..1a549c96 100644 --- a/OptiScaler/hooks/LibraryLoad_Hooks.cpp +++ b/OptiScaler/hooks/LibraryLoad_Hooks.cpp @@ -1129,13 +1129,14 @@ void LibraryLoadHooks::CheckModulesInMemory() // try_lock: this runs on every load, including nested ones from threads already inside the loader. // A busy mutex means another thread is hooking that plugin, and the next call catches anything missed. + // Modules are looked up before locking, GetModuleHandle/GetModuleFileName can take the loader lock. + // The unlocked is*Hooked checks only skip the lookups, they are checked again under the lock. + if (!StreamlineHooks::isDlssHooked()) { - std::unique_lock lock(StreamlineHooks::mutexHookDlss, std::try_to_lock); - if (lock.owns_lock() && !StreamlineHooks::isDlssHooked()) + if (HMODULE slDlss = GetDllNameWModule(&slDlssNamesW)) { - HMODULE slDlss = nullptr; - slDlss = GetDllNameWModule(&slDlssNamesW); - if (slDlss != nullptr) + std::unique_lock lock(StreamlineHooks::mutexHookDlss, std::try_to_lock); + if (lock.owns_lock() && !StreamlineHooks::isDlssHooked()) { LOG_DEBUG("sl.dlss.dll already in memory"); StreamlineHooks::hookDlss(slDlss); @@ -1143,12 +1144,12 @@ void LibraryLoadHooks::CheckModulesInMemory() } } + if (!StreamlineHooks::isDlssdHooked()) { - std::unique_lock lock(StreamlineHooks::mutexHookDlssd, std::try_to_lock); - if (lock.owns_lock() && !StreamlineHooks::isDlssdHooked()) + if (HMODULE slDlssd = GetDllNameWModule(&slDlssdNamesW)) { - HMODULE slDlssd = GetDllNameWModule(&slDlssdNamesW); - if (slDlssd != nullptr) + std::unique_lock lock(StreamlineHooks::mutexHookDlssd, std::try_to_lock); + if (lock.owns_lock() && !StreamlineHooks::isDlssdHooked()) { LOG_DEBUG("sl.dlss_d.dll already in memory"); StreamlineHooks::hookDlssd(slDlssd); @@ -1157,18 +1158,18 @@ void LibraryLoadHooks::CheckModulesInMemory() } // DLSS-G + if (!StreamlineHooks::isDlssgHooked() || !StreamlineHooks::isLocalDlssgHooked()) { - std::unique_lock lock1(StreamlineHooks::mutexHookDlssg, std::try_to_lock); - std::unique_lock lock2(StreamlineHooks::mutexHookLocalDlssg, std::try_to_lock); - if (lock1.owns_lock() && lock2.owns_lock() && - (!StreamlineHooks::isDlssgHooked() || !StreamlineHooks::isLocalDlssgHooked())) + HMODULE slDlssg = GetDllNameWModule(&slDlssgNamesW); + + if (slDlssg != nullptr && slDlssg != State::Instance().optiSlDLSSG) { - HMODULE slDlssg = GetDllNameWModule(&slDlssgNamesW); + const bool localDlssg = isLocalStreamlineModule(slDlssg); - if (slDlssg != nullptr && slDlssg != State::Instance().optiSlDLSSG) + std::unique_lock lock1(StreamlineHooks::mutexHookDlssg, std::try_to_lock); + std::unique_lock lock2(StreamlineHooks::mutexHookLocalDlssg, std::try_to_lock); + if (lock1.owns_lock() && lock2.owns_lock()) { - const bool localDlssg = isLocalStreamlineModule(slDlssg); - if (localDlssg && State::Instance().activeFgOutput == FGOutput::DLSSG) { if (!StreamlineHooks::isLocalDlssgHooked()) @@ -1187,17 +1188,18 @@ void LibraryLoadHooks::CheckModulesInMemory() } // Reflex + if (!StreamlineHooks::isReflexHooked()) { - std::unique_lock lock(StreamlineHooks::mutexHookReflex, std::try_to_lock); - if (lock.owns_lock() && !StreamlineHooks::isReflexHooked()) + HMODULE slReflex = GetDllNameWModule(&slReflexNamesW); + + if (slReflex != nullptr && slReflex != State::Instance().optiSlReflex) { - HMODULE slReflex = GetDllNameWModule(&slReflexNamesW); + const bool localReflex = isLocalStreamlineModule(slReflex); - if (slReflex != nullptr && slReflex != State::Instance().optiSlReflex) + if (!localReflex || State::Instance().activeFgOutput != FGOutput::DLSSG) { - const bool localReflex = isLocalStreamlineModule(slReflex); - - if (!localReflex || State::Instance().activeFgOutput != FGOutput::DLSSG) + std::unique_lock lock(StreamlineHooks::mutexHookReflex, std::try_to_lock); + if (lock.owns_lock() && !StreamlineHooks::isReflexHooked()) { if (localReflex) LOG_DEBUG("local sl.reflex.dll already in memory"); @@ -1211,17 +1213,18 @@ void LibraryLoadHooks::CheckModulesInMemory() } // PCL + if (!StreamlineHooks::isPclHooked()) { - std::unique_lock lock(StreamlineHooks::mutexHookPcl, std::try_to_lock); - if (lock.owns_lock() && !StreamlineHooks::isPclHooked()) + HMODULE slPcl = GetDllNameWModule(&slPclNamesW); + + if (slPcl != nullptr && slPcl != State::Instance().optiSlPCL) { - HMODULE slPcl = GetDllNameWModule(&slPclNamesW); + const bool localPcl = isLocalStreamlineModule(slPcl); - if (slPcl != nullptr && slPcl != State::Instance().optiSlPCL) + if (!localPcl || State::Instance().activeFgOutput != FGOutput::DLSSG) { - const bool localPcl = isLocalStreamlineModule(slPcl); - - if (!localPcl || State::Instance().activeFgOutput != FGOutput::DLSSG) + std::unique_lock lock(StreamlineHooks::mutexHookPcl, std::try_to_lock); + if (lock.owns_lock() && !StreamlineHooks::isPclHooked()) { if (localPcl) LOG_DEBUG("local sl.pcl.dll already in memory"); @@ -1234,13 +1237,14 @@ void LibraryLoadHooks::CheckModulesInMemory() } } + if (!StreamlineHooks::isCommonHooked()) { - std::unique_lock lock(StreamlineHooks::mutexHookCommon, std::try_to_lock); - if (lock.owns_lock() && !StreamlineHooks::isCommonHooked()) + HMODULE slCommon = GetDllNameWModule(&slCommonNamesW); + + if (slCommon != nullptr && slCommon != State::Instance().optiSlCommon) { - HMODULE slCommon = nullptr; - slCommon = GetDllNameWModule(&slCommonNamesW); - if (slCommon != nullptr && slCommon != State::Instance().optiSlCommon) + std::unique_lock lock(StreamlineHooks::mutexHookCommon, std::try_to_lock); + if (lock.owns_lock() && !StreamlineHooks::isCommonHooked()) { LOG_DEBUG("sl.common.dll already in memory"); StreamlineHooks::hookCommon(slCommon); diff --git a/OptiScaler/hooks/Streamline_Hooks.cpp b/OptiScaler/hooks/Streamline_Hooks.cpp index 02b5f155..4bb84c29 100644 --- a/OptiScaler/hooks/Streamline_Hooks.cpp +++ b/OptiScaler/hooks/Streamline_Hooks.cpp @@ -20,6 +20,7 @@ #include #include +#include #pragma intrinsic(_ReturnAddress) static std::wstring GetModulePath(HMODULE module) @@ -32,6 +33,41 @@ static std::wstring GetModulePath(HMODULE module) return std::wstring(buffer, len); } +// Threads holding the loader lock wait for the plugin mutexes (LoadLibraryCheckW of a nested load), so code holding +// one must not take the loader lock. GetModuleFileNameW and GetProcAddress can (always under Wine), these two don't. + +// NT path of the file mapped at address, empty if nothing is mapped there +static std::wstring GetMappedPath(const void* address) +{ + wchar_t buffer[MAX_PATH * 2] {}; + const auto len = K32GetMappedFileNameW(GetCurrentProcess(), const_cast(address), buffer, + static_cast(std::size(buffer))); + return std::wstring(buffer, len); +} + +// Reads the export table directly +static void* FindExport(HMODULE module, const char* name) +{ + struct Context + { + const char* name; + void* code; + } context { name, nullptr }; + + DetourEnumerateExports(module, &context, + [](PVOID ctx, ULONG, LPCSTR exportName, PVOID code) -> BOOL + { + auto c = static_cast(ctx); + if (exportName == nullptr || strcmp(exportName, c->name) != 0) + return TRUE; + + c->code = code; + return FALSE; + }); + + return context.code; +} + // Plugin may fail (denylist etc.) and leave pluginJSON empty static bool HasPluginJson(bool result, const char** pluginJSON) { @@ -2528,7 +2564,7 @@ bool StreamlineHooks::isSlotAlive(const PluginHookSlot& slot) if (slot.module == nullptr || slot.target == nullptr || slot.path.empty()) return false; - const auto current = GetModulePath(slot.module); + const auto current = GetMappedPath(slot.module); if (current.empty() || _wcsicmp(current.c_str(), slot.path.c_str()) != 0) return false; @@ -2674,8 +2710,7 @@ void StreamlineHooks::hookPlugin(SlPlugin plugin, HMODULE module) } } - auto target = - reinterpret_cast(KernelBaseProxy::GetProcAddress_()(module, "slGetPluginFunction")); + auto target = reinterpret_cast(FindExport(module, "slGetPluginFunction")); if (target == nullptr) { @@ -2731,7 +2766,7 @@ void StreamlineHooks::hookPlugin(SlPlugin plugin, HMODULE module) slot.target = reinterpret_cast(target); slot.module = module; slot.imageSize = imageSize; - slot.path = GetModulePath(module); + slot.path = GetMappedPath(module); memcpy(slot.patchBytes, slot.target, kPatchBytes); activePluginModule(plugin) = module; diff --git a/OptiScaler/hooks/Streamline_Hooks.h b/OptiScaler/hooks/Streamline_Hooks.h index e9726b91..1df0218f 100644 --- a/OptiScaler/hooks/Streamline_Hooks.h +++ b/OptiScaler/hooks/Streamline_Hooks.h @@ -152,6 +152,8 @@ class StreamlineHooks // so hooking a new copy must never remove the hook from a copy that is still loaded. // A slot is released only after the loader really unmapped its module (see onModuleFreed). // Callers of hook*/unhook* must hold the matching mutexHook* (same as before). + // A thread holding the loader lock can wait for a mutexHook*, so never take the loader lock while holding one: + // no GetModuleHandle/GetModuleFileName/GetProcAddress/LoadLibrary, look those up before locking. static void unhookDlss(); static void hookDlss(HMODULE slDlss); @@ -225,7 +227,7 @@ class StreamlineHooks void* target; // slGetPluginFunction inside the plugin HMODULE module; size_t imageSize; - std::wstring path; + std::wstring path; // NT path of the mapped file (GetMappedFileNameW) uint8_t patchBytes[kPatchBytes]; // bytes at target right after attaching, to detect a fresh image };