Double texture release patch for 1.6.1179+

SKSE/Plugins/EnderalSE.ini

@@ -10,3 +10,4 @@ AutoScaleHeroMenu = true
 WarnFormTypeCollisions = true
 MenuAspectRatioFix = true
 LoadScreenFrameFix = true
+TextureDoubleReleaseFix = true

source/Enderal DLL/src/Main.cpp

@@ -17,6 +17,7 @@
 #include "Patches/FormTypeCollisionDetector.h"
 #include "Patches/ForceLanguage.h"
 #include "Patches/MenuAspectRatioFix.h"
+#include "Patches/TextureDoubleReleaseFix.h"
 
 using namespace SKSE;
 
@@ -32,7 +33,8 @@ static std::map<std::string, bool> g_settings{
 	{ "AttachLightHitEffectCrashFix", true },
 	{ "AutoScaleHeroMenu", true },
 	{ "MenuAspectRatioFix", true },
-	{ "LoadScreenFrameFix", true }
+	{ "LoadScreenFrameFix", true },
+	{ "TextureDoubleReleaseFix", true }
 };
 
 namespace {
@@ -258,6 +260,9 @@ SKSEPluginLoad(const LoadInterface* skse) {
 			MenuAspectRatioFix::Install();
 		}
 		LoadScreenFrameFix::Install(g_settings.at("LoadScreenFrameFix"));
+		if (g_settings.at("TextureDoubleReleaseFix")) {
+			TextureDoubleReleaseFix::Install();
+		}
 	}
 
     logger::info("{} has finished loading.", plugin->GetName());

source/Enderal DLL/src/Patches/TextureDoubleReleaseFix.h

@@ -0,0 +1,63 @@
+#pragma once
+
+// From 1.6.1179 on, the renderer function that frees a BSGraphics::Texture (AE id 77322) calls
+// Release() twice on each of its three D3D11 objects: once in the original sequence, then again
+// in what looks like an inlined destructor. The texture loader takes one reference per object,
+// so the second batch underflows. D3D11 usually defers the destruction of an object that a
+// pending command list still references, which is why the second call normally lands on live
+// memory. When the runtime frees the object at once, the game crashes in this function with
+// `call [rax+0x10]` on a zeroed vtable. The patch jumps over the second batch.
+namespace TextureDoubleReleaseFix
+{
+	// Everything before the deallocation call. Only rel8 branches inside, so these bytes are
+	// identical on 1.6.1179, 1.7.99 and 1.7.104.
+	constexpr std::uint8_t DOUBLE_RELEASE_FORM[]{
+		0x40, 0x53,                                                  // push rbx
+		0x48, 0x83, 0xEC, 0x30,                                      // sub rsp,0x30
+		0x48, 0xC7, 0x44, 0x24, 0x20, 0xFE, 0xFF, 0xFF, 0xFF,        // mov [rsp+0x20],-2
+		0x48, 0x8B, 0xDA,                                            // mov rbx,rdx
+		0xB8, 0xFF, 0xFF, 0xFF, 0xFF,                                // mov eax,-1
+		0xF0, 0x0F, 0xC1, 0x42, 0x20,                                // lock xadd [rdx+0x20],eax
+		0x83, 0xF8, 0x01,                                            // cmp eax,1
+		0x75, 0x6C,                                                  // jne +0x8D (return)
+		0x48, 0x8B, 0x4A, 0x10, 0x48, 0x85, 0xC9, 0x74, 0x06, 0x48, 0x8B, 0x01, 0xFF, 0x50, 0x10,  // Release [rdx+0x10]
+		0x48, 0x8B, 0x0B, 0x48, 0x85, 0xC9, 0x74, 0x06, 0x48, 0x8B, 0x01, 0xFF, 0x50, 0x10,        // Release [rbx]
+		0x48, 0x8B, 0x4B, 0x08, 0x48, 0x85, 0xC9, 0x74, 0x07, 0x48, 0x8B, 0x01, 0xFF, 0x50, 0x10,  // Release [rbx+0x08]
+		0x90,                                                        // nop
+		0x48, 0x8B, 0x0B, 0x48, 0x85, 0xC9, 0x74, 0x06, 0x48, 0x8B, 0x01, 0xFF, 0x50, 0x10,        // +0x4E: Release [rbx] again
+		0x48, 0x8B, 0x4B, 0x08, 0x48, 0x85, 0xC9, 0x74, 0x06, 0x48, 0x8B, 0x01, 0xFF, 0x50, 0x10,  // Release [rbx+0x08] again
+		0x48, 0x8B, 0x4B, 0x10, 0x48, 0x85, 0xC9, 0x74, 0x07, 0x48, 0x8B, 0x01, 0xFF, 0x50, 0x10,  // Release [rbx+0x10] again
+		0x90                                                         // nop, +0x7B follows: mov edx,0x28
+	};
+
+	// 1.6.1170 and older release each object once. Recognized so the log can say so.
+	constexpr std::uint8_t SINGLE_RELEASE_PROLOGUE[]{ 0x40, 0x53, 0x48, 0x83, 0xEC, 0x20, 0x48, 0x8B, 0xDA };
+
+	constexpr std::size_t SECOND_BATCH = 0x4E;
+	constexpr std::size_t DEALLOCATE = 0x7B;
+
+	void Install()
+	{
+		if (!REL::Module::IsAE()) {
+			return;
+		}
+
+		const auto target = REL::Relocation<std::uintptr_t>{ REL::ID(77322) }.address();
+		const auto bytes = reinterpret_cast<const std::uint8_t*>(target);
+
+		// jmp short +0x2B, from +0x50 to +0x7B
+		const std::uint8_t code[]{ 0xEB, static_cast<std::uint8_t>(DEALLOCATE - SECOND_BATCH - 2) };
+
+		if (std::memcmp(bytes, DOUBLE_RELEASE_FORM, sizeof(DOUBLE_RELEASE_FORM)) == 0) {
+			logger::info("Patching the double texture release...");
+			REL::safe_write(target + SECOND_BATCH, code, sizeof(code));
+		} else if (std::memcmp(bytes, DOUBLE_RELEASE_FORM, SECOND_BATCH) == 0 && std::memcmp(bytes + SECOND_BATCH, code, sizeof(code)) == 0) {
+			// Another plugin wrote the same jump first. Nothing left to do.
+			logger::info("Double texture release already patched by another plugin");
+		} else if (std::memcmp(bytes, SINGLE_RELEASE_PROLOGUE, sizeof(SINGLE_RELEASE_PROLOGUE)) == 0) {
+			logger::info("Texture release function releases once, no patch needed");
+		} else {
+			logger::error("Unexpected bytes in the texture release function, skipping the double release patch...");
+		}
+	}
+}