Refuse late device-attach under ARM64 emulation

Hot-patching an already-live device's vtable methods via SetDevice races
the emulator's self-modifying-code handling (Windows-on-ARM's xtajit, or
Wine on FEX-Emu/box64). Detect that case with IsWow64Process2 and refuse
the attach instead, so callers fall back to loading gMod as d3d9.dll
before the game creates its device.

header/Error.h

@@ -3,6 +3,7 @@
 // define return values, a value less than zero indicates an error
 #define RETURN_OK 0
 #define RETURN_EXISTS -70
+#define RETURN_UNSUPPORTED_UNDER_EMULATION -71
 
 #define RETURN_FATAL_ERROR -1
 #define RETURN_NO_MEMORY -2

header/Utils.h

@@ -25,4 +25,23 @@ namespace utils {
 
         return wstr;
     }
+
+    // True when this x86/x64 process is running under an ARM64 emulator (Windows-on-ARM's
+    // xtajit, or Wine on FEX-Emu/box64 on ARM64 Linux). Patching a method on a device the
+    // game is already calling races that emulator's self-modifying-code handling, so callers
+    // must not hot-attach to an existing device in that case.
+    inline bool IsRunningUnderArm64Emulation()
+    {
+        using IsWow64Process2_t = BOOL(WINAPI*)(HANDLE, USHORT*, USHORT*);
+        const auto IsWow64Process2_fn = reinterpret_cast<IsWow64Process2_t>(
+            GetProcAddress(GetModuleHandleA("kernel32.dll"), "IsWow64Process2"));
+        if (!IsWow64Process2_fn) return false;
+
+        USHORT process_machine = IMAGE_FILE_MACHINE_UNKNOWN;
+        USHORT native_machine = IMAGE_FILE_MACHINE_UNKNOWN;
+        if (!IsWow64Process2_fn(GetCurrentProcess(), &process_machine, &native_machine))
+            return false;
+
+        return native_machine == IMAGE_FILE_MACHINE_ARM64 && process_machine != IMAGE_FILE_MACHINE_UNKNOWN;
+    }
 }

source/dll_main.cpp

@@ -264,6 +264,14 @@ void InitInstance(HINSTANCE hModule)
 extern "C" __declspec(dllexport) int __cdecl SetDevice(IDirect3DDevice9* device)
 {
     if (!device) return RETURN_BAD_ARGUMENT;
+    if (utils::IsRunningUnderArm64Emulation()) {
+        // The device is already live and being called every frame; hot-patching its vtable
+        // methods here would race the emulator's self-modifying-code handling. Load gMod as
+        // d3d9.dll before the game starts instead, so hooks install via Direct3DCreate9(Ex)
+        // on a device nobody has called into yet.
+        Warning("SetDevice: late device-attach isn't supported under ARM64 emulation\n");
+        return RETURN_UNSUPPORTED_UNDER_EMULATION;
+    }
     try {
         return RegisterExistingDevice(device) ? RETURN_OK : RETURN_EXISTS;
     }