From 92c6b115f3c600724a413e912154520b0d928a0e Mon Sep 17 00:00:00 2001 From: Marc Date: Wed, 10 Jun 2026 15:13:00 +0000 Subject: [PATCH] Make D3D9 detours fall through after unhook to avoid teardown crash Keep the o_* trampoline pointers valid on RemoveAllD3D9Hooks and add a g_unhooked flag each detour checks after calling the original, so a surviving vtable patch invoked during/after teardown just returns the real result instead of touching freed TextureClient state. Tear down off the loader lock via a new Shutdown export and skip teardown on process exit. Co-Authored-By: Claude Opus 4.8 (1M context) --- source/D3D9Hooks.cpp | 36 +++++++++++++++++++++++------------- source/dll_main.cpp | 22 +++++++++++++++++++--- 2 files changed, 42 insertions(+), 16 deletions(-) diff --git a/source/D3D9Hooks.cpp b/source/D3D9Hooks.cpp index fe3ec89..9fe89c3 100644 --- a/source/D3D9Hooks.cpp +++ b/source/D3D9Hooks.cpp @@ -2,6 +2,7 @@ #include "D3D9Hooks.h" #include "MinHook.h" +#include #include #include #include @@ -72,6 +73,9 @@ namespace { std::vector g_hooked_targets; + // Set when teardown begins; a detour still reached after this just calls the original. + std::atomic g_unhooked{false}; + // device -> owning TextureClient std::mutex g_devices_mutex; std::unordered_map g_devices; @@ -174,6 +178,7 @@ namespace { D3DPRESENT_PARAMETERS* pPresentationParameters, IDirect3DDevice9** ppReturnedDeviceInterface) { const HRESULT hr = o_CreateDevice(self, Adapter, DeviceType, hFocusWindow, BehaviorFlags, pPresentationParameters, ppReturnedDeviceInterface); + if (g_unhooked) return hr; if (SUCCEEDED(hr) && ppReturnedDeviceInterface && *ppReturnedDeviceInterface) { OnDeviceCreated(*ppReturnedDeviceInterface); } @@ -184,6 +189,7 @@ namespace { D3DPRESENT_PARAMETERS* pPresentationParameters, D3DDISPLAYMODEEX* pFullscreenDisplayMode, IDirect3DDevice9Ex** ppReturnedDeviceInterface) { const HRESULT hr = o_CreateDeviceEx(self, Adapter, DeviceType, hFocusWindow, BehaviorFlags, pPresentationParameters, pFullscreenDisplayMode, ppReturnedDeviceInterface); + if (g_unhooked) return hr; if (SUCCEEDED(hr) && ppReturnedDeviceInterface && *ppReturnedDeviceInterface) { OnDeviceCreated(*ppReturnedDeviceInterface); } @@ -193,6 +199,7 @@ namespace { ULONG STDMETHODCALLTYPE h_DeviceRelease(IDirect3DDevice9* self) { const ULONG count = o_DeviceRelease(self); + if (g_unhooked) return count; if (count == 0) { TextureClient* client = nullptr; { @@ -212,6 +219,7 @@ namespace { IDirect3DTexture9** ppTexture, HANDLE* pSharedHandle) { const HRESULT hr = o_CreateTexture(self, Width, Height, Levels, Usage, Format, Pool, ppTexture, pSharedHandle); + if (g_unhooked) return hr; if (SUCCEEDED(hr) && ppTexture && *ppTexture) { InstallTextureReleaseHook(*ppTexture, TexType::Tex2D); if (auto* client = ClientFor(self)) @@ -224,6 +232,7 @@ namespace { IDirect3DVolumeTexture9** ppVolumeTexture, HANDLE* pSharedHandle) { const HRESULT hr = o_CreateVolumeTexture(self, Width, Height, Depth, Levels, Usage, Format, Pool, ppVolumeTexture, pSharedHandle); + if (g_unhooked) return hr; if (SUCCEEDED(hr) && ppVolumeTexture && *ppVolumeTexture) { InstallTextureReleaseHook(*ppVolumeTexture, TexType::Volume); if (auto* client = ClientFor(self)) @@ -236,6 +245,7 @@ namespace { IDirect3DCubeTexture9** ppCubeTexture, HANDLE* pSharedHandle) { const HRESULT hr = o_CreateCubeTexture(self, EdgeLength, Levels, Usage, Format, Pool, ppCubeTexture, pSharedHandle); + if (g_unhooked) return hr; if (SUCCEEDED(hr) && ppCubeTexture && *ppCubeTexture) { InstallTextureReleaseHook(*ppCubeTexture, TexType::Cube); if (auto* client = ClientFor(self)) @@ -248,6 +258,7 @@ namespace { { // Pass the real textures through, then re-evaluate mods on the changed destination. const HRESULT hr = o_UpdateTexture(self, pSourceTexture, pDestinationTexture); + if (g_unhooked) return hr; if (SUCCEEDED(hr)) { if (auto* client = ClientFor(self)) client->OnUpdateTexture(pSourceTexture, pDestinationTexture); @@ -257,6 +268,7 @@ namespace { HRESULT STDMETHODCALLTYPE h_BeginScene(IDirect3DDevice9* self) { + if (g_unhooked) return o_BeginScene(self); if (auto* client = ClientFor(self)) client->OnBeginScene(); return o_BeginScene(self); @@ -264,6 +276,7 @@ namespace { HRESULT STDMETHODCALLTYPE h_SetTexture(IDirect3DDevice9* self, DWORD Stage, IDirect3DBaseTexture9* pTexture) { + if (g_unhooked) return o_SetTexture(self, Stage, pTexture); IDirect3DBaseTexture9* bind = pTexture; if (auto* client = ClientFor(self)) bind = client->ResolveBinding(pTexture); // substitute the fake when modded @@ -273,6 +286,7 @@ namespace { HRESULT STDMETHODCALLTYPE h_GetTexture(IDirect3DDevice9* self, DWORD Stage, IDirect3DBaseTexture9** ppTexture) { const HRESULT hr = o_GetTexture(self, Stage, ppTexture); + if (g_unhooked) return hr; if (SUCCEEDED(hr) && ppTexture && *ppTexture) { if (auto* client = ClientFor(self)) { if (auto* original = client->ResolveOriginalFromFake(*ppTexture)) { @@ -295,6 +309,7 @@ namespace { ULONG STDMETHODCALLTYPE h_Tex2DRelease(IUnknown* self) { const ULONG count = o_Tex2DRelease(self); + if (g_unhooked) return count; ReleaseTextureCleanup(self, count); return count; } @@ -302,6 +317,7 @@ namespace { ULONG STDMETHODCALLTYPE h_VolumeRelease(IUnknown* self) { const ULONG count = o_VolumeRelease(self); + if (g_unhooked) return count; ReleaseTextureCleanup(self, count); return count; } @@ -309,6 +325,7 @@ namespace { ULONG STDMETHODCALLTYPE h_CubeRelease(IUnknown* self) { const ULONG count = o_CubeRelease(self); + if (g_unhooked) return count; ReleaseTextureCleanup(self, count); return count; } @@ -317,6 +334,7 @@ namespace { void InstallD3D9Hooks(IDirect3D9* d3d9, const bool is_ex) { if (d3d9 == nullptr || g_d3d9_hooks_installed) return; + g_unhooked = false; // re-arm in case a prior teardown left the flag set void** vt = GetVTable(d3d9); Hook(vt[kIDirect3D9_CreateDevice], &h_CreateDevice, reinterpret_cast(&o_CreateDevice)); if (is_ex) { @@ -333,12 +351,16 @@ bool RegisterExistingDevice(IDirect3DDevice9* device) if (g_devices.contains(device)) return false; // already known if (!g_devices.empty()) return false; // gMod supports a single device at a time } + g_unhooked = false; // re-arm in case a prior teardown left the flag set OnDeviceCreated(device); // installs device hooks + TextureClient (idempotent) return ClientFor(device) != nullptr; } void RemoveAllD3D9Hooks() { + // Signal first: a detour reached after this (e.g. a surviving patch) falls through. + g_unhooked = true; + for (void* target : g_hooked_targets) { MH_DisableHook(target); MH_RemoveHook(target); @@ -351,19 +373,7 @@ void RemoveAllD3D9Hooks() g_volume_release_installed = false; g_cube_release_installed = false; - o_CreateDevice = nullptr; - o_CreateDeviceEx = nullptr; - o_DeviceRelease = nullptr; - o_CreateTexture = nullptr; - o_CreateVolumeTexture = nullptr; - o_CreateCubeTexture = nullptr; - o_UpdateTexture = nullptr; - o_BeginScene = nullptr; - o_SetTexture = nullptr; - o_GetTexture = nullptr; - o_Tex2DRelease = nullptr; - o_VolumeRelease = nullptr; - o_CubeRelease = nullptr; + // Leave the o_* trampoline pointers intact so a surviving detour can still call through. } void DestroyAllTextureClients() diff --git a/source/dll_main.cpp b/source/dll_main.cpp index a98fac2..d68bf3e 100644 --- a/source/dll_main.cpp +++ b/source/dll_main.cpp @@ -2,6 +2,7 @@ #include #include "MinHook.h" #include "D3D9Hooks.h" +#include import TextureClient; @@ -194,9 +195,12 @@ BOOL WINAPI DllMain(HINSTANCE hModule, DWORD ul_reason_for_call, LPVOID lpReserv break; } case DLL_PROCESS_DETACH: { - // lpReserved == nullptr means FreeLibrary (device still alive); non-null - // means process exit, where d3d9/the device may already be gone. - ExitInstance(lpReserved == nullptr); + // Process exit (lpReserved != nullptr): other threads are gone and the OS + // reclaims everything, so touching MinHook/the device here only risks a hang. + if (lpReserved != nullptr) + break; + // FreeLibrary unload: the device is still live, so tear down cleanly. + ExitInstance(true); break; } default: break; @@ -314,8 +318,20 @@ extern "C" __declspec(dllexport) int __cdecl GetFiles(wchar_t* buffer, const siz } } +// Optional clean-shutdown entry: call this from the host (on a normal thread) BEFORE +// FreeLibrary so teardown runs off the loader lock, where MinHook can safely suspend +// the render thread. A later FreeLibrary then unloads with nothing left to undo. +extern "C" __declspec(dllexport) void __cdecl Shutdown() +{ + ExitInstance(true); +} + void ExitInstance(bool is_unloading) { + // Teardown must run exactly once, whether reached via Shutdown() or DllMain detach. + static std::atomic torn_down{false}; + if (torn_down.exchange(true)) return; + DISABLE_HOOK(GetProcAddress_fn); // Revert every D3D9 vtable hook so the original objects are left pristine.