Handle blocking dialogs with DLL, replaced dialoguemenu.swf with unchanged Better Dialogue Controls

source/Enderal DLL/src/DialogueMenuPatch.h

@@ -0,0 +1,47 @@
+#pragma once
+
+inline bool bTabBlocked = false;
+
+class DialogueMenuPatch final : public RE::DialogueMenu
+{
+public:
+	static void Install()
+	{
+		REL::Relocation<std::uintptr_t> OpenJournalMenuFunc{ 0x0092412C };
+
+		REL::Relocation<uintptr_t> vtbl(RE::VTABLE_DialogueMenu[0]);
+		_ProcessMessageFn = vtbl.write_vfunc(0x4, &ProcessMessageEx);
+	}
+
+	static void BlockTab(bool a_block = true)
+	{
+		bTabBlocked = a_block;
+	}
+
+private:
+	RE::UI_MESSAGE_RESULTS ProcessMessageEx(RE::UIMessage& a_message)
+	{
+		if (bTabBlocked && a_message.type == RE::UI_MESSAGE_TYPE::kScaleformEvent) {
+			RE::BSUIScaleformData* data = static_cast<RE::BSUIScaleformData*>(a_message.data);
+
+			if (data && data->scaleformEvent->type == RE::GFxEvent::EventType::kKeyDown) {
+				RE::GFxKeyEvent* key = (RE::GFxKeyEvent*)data->scaleformEvent;
+
+				// Escape triggers a kTab event, added kEscape just in case
+				if (key && (key->keyCode == RE::GFxKey::kTab || key->keyCode == RE::GFxKey::kEscape) && !RE::UI::GetSingleton()->IsMenuOpen(RE::ContainerMenu::MENU_NAME) && !RE::UI::GetSingleton()->IsMenuOpen(RE::BarterMenu::MENU_NAME)) {
+					// The best solution is to show the quest journal, like in LE, but showing a journal seems to mess with framerate of the UI, see dialogue menu after showing journal
+					//RE::UIMessageQueue::GetSingleton()->AddMessage(RE::InterfaceStrings::GetSingleton()->journalMenu, RE::UI_MESSAGE_TYPE::kShow, 0i64);
+					RE::BSTSmartPointer<RE::BSScript::IStackCallbackFunctor> stackCallback;
+					RE::BSScript::Internal::VirtualMachine::GetSingleton()->DispatchStaticCall("_00E_ConfirmDialogQuit", "Show", RE::MakeFunctionArguments(), stackCallback);
+					return RE::UI_MESSAGE_RESULTS::kIgnore;
+				}
+			}
+		}
+
+		return _ProcessMessageFn(this, a_message);
+	}
+
+	using ProcessMessageFn = decltype(&RE::DialogueMenu::ProcessMessage);
+
+	inline static REL::Relocation<ProcessMessageFn> _ProcessMessageFn;
+};

source/Enderal DLL/src/EventListener.cpp

@@ -1,5 +1,6 @@
 #include "EventListener.h"
 #include "Util.h"
+#include "DialogueMenuPatch.h"
 #include <shellapi.h>
 
 auto EventListener::GetSingleton() -> EventListener*
@@ -11,8 +12,9 @@ auto EventListener::GetSingleton() -> EventListener*
 // Unused
 void EventListener::Install()
 {
-	RE::ScriptEventSourceHolder::GetSingleton()->AddEventSink<RE::TESContainerChangedEvent>(EventListener::GetSingleton());
+	//RE::ScriptEventSourceHolder::GetSingleton()->AddEventSink<RE::TESContainerChangedEvent>(EventListener::GetSingleton());
 	RE::UI::GetSingleton()->AddEventSink<RE::MenuOpenCloseEvent>(EventListener::GetSingleton());
+	SKSE::GetModCallbackEventSource()->AddEventSink(EventListener::GetSingleton());
 }
 
 auto EventListener::ProcessEvent(
@@ -46,21 +48,29 @@ auto EventListener::ProcessEvent(
 	RE::BSTEventSource<RE::MenuOpenCloseEvent>* a_eventSource)
 	-> RE::BSEventNotifyControl
 {
-	if (a_event->opening && a_event->menuName == RE::JournalMenu::MENU_NAME) {
-		if (RE::UI::GetSingleton()->IsMenuOpen(RE::DialogueMenu::MENU_NAME)) {
-			const auto movie = RE::UI::GetSingleton()->GetMovieView(RE::JournalMenu::MENU_NAME);
-			if (movie) {
-				// skse.OpenMenu("Journal Menu") opens quest journal uninitialized, without any tab active and hotkeys bound
-				// We have to activate a tab for it to work properly
-				std::array<RE::GFxValue, 2> args;
-				args[0].SetNumber(2);
-				args[1].SetBoolean(true);
-				movie->Invoke("_root.QuestJournalFader.Menu_mc.RestoreSavedSettings", nullptr, args.data(), static_cast<std::uint32_t>(args.size()));
+	if (a_event->opening) {
+		if (a_event->menuName == RE::JournalMenu::MENU_NAME) {
+			if (RE::UI::GetSingleton()->IsMenuOpen(RE::DialogueMenu::MENU_NAME)) {
+				const auto movie = RE::UI::GetSingleton()->GetMovieView(RE::JournalMenu::MENU_NAME);
+				if (movie) {
+					// UIMessage opens quest journal uninitialized, without any tab active and hotkeys bound
+					// We have to activate a tab for it to work properly
+					std::array<RE::GFxValue, 2> args;
+					args[0].SetNumber(2);
+					args[1].SetBoolean(false);
+					movie->Invoke("_root.QuestJournalFader.Menu_mc.RestoreSavedSettings", nullptr, args.data(), static_cast<std::uint32_t>(args.size()));
+				}
 			}
+		} else if (a_event->menuName == RE::DialogueMenu::MENU_NAME) {
+			// Make sure Tab is not blocked. If it should be, a Papyrus script will block it a few frames later.
+			DialogueMenuPatch::BlockTab(false);
+		}
+	} else {
+		if (a_event->menuName == RE::DialogueMenu::MENU_NAME) {
+			DialogueMenuPatch::BlockTab(false);
+		} else if (a_event->menuName == RE::MainMenu::MENU_NAME) {
+			CheckScriptVersions();
 		}
-	} else if (!a_event->opening && a_event->menuName == RE::MainMenu::MENU_NAME) {
-		RE::UI::GetSingleton()->RemoveEventSink<RE::MenuOpenCloseEvent>(EventListener::GetSingleton());
-		CheckScriptVersions();
 	}
 
 	return RE::BSEventNotifyControl::kContinue;

source/Enderal DLL/src/Main.cpp

@@ -6,6 +6,7 @@
 #include "MapMarkerPlacement.h"
 #include "AchievementFix.h"
 #include "BinkInterruptPatch.h"
+#include "DialogueMenuPatch.h"
 
 using namespace SKSE;
 
@@ -158,11 +159,12 @@ SKSEPluginLoad(const LoadInterface* skse) {
 	setting->data.s = nullptr;
 	RE::INISettingCollection::GetSingleton()->WriteSetting(setting);
 
-	SKSE::GetModCallbackEventSource()->AddEventSink(EventListener::GetSingleton());
-	RE::UI::GetSingleton()->AddEventSink<RE::MenuOpenCloseEvent>(EventListener::GetSingleton());
+	EventListener::Install();
 
 	GetPapyrusInterface()->Register(Papyrus::Bind);
 
+	DialogueMenuPatch::Install();
+
 	if (g_settings.at("VideoInterruptPatch")) {
 		logger::info("Making videos interruptible...");
 		BinkInterruptPatch::Install();

source/Enderal DLL/src/PapyrusFunctions.h

@@ -2,6 +2,7 @@
 
 #include "Util.h"
 #include "PersistentFormManager.h"
+#include "DialogueMenuPatch.h"
 
 namespace Papyrus::PapyrusFunctions
 {
@@ -87,6 +88,16 @@ namespace Papyrus::PapyrusFunctions
 		return sstream.str();
 	}
 
+	void DisableDialogueQuitting(RE::StaticFunctionTag*)
+	{
+		DialogueMenuPatch::BlockTab();
+	}
+
+	void EnableDialogueQuitting(RE::StaticFunctionTag*)
+	{
+		DialogueMenuPatch::BlockTab(false);
+	}
+
 	inline void Bind(VM& a_vm)
 	{
 		BIND(CreatePotion);
@@ -101,5 +112,9 @@ namespace Papyrus::PapyrusFunctions
 		logger::info("{}", "Registered GetPlayerHash"sv);
 		BIND(StringToHex);
 		logger::info("{}", "Registered StringToHex"sv);
+		BIND(DisableDialogueQuitting);
+		logger::info("{}", "Registered DisableDialogueQuitting"sv);
+		BIND(EnableDialogueQuitting);
+		logger::info("{}", "Registered EnableDialogueQuitting"sv);
 	}
 }

source/scripts/_00e_confirmdialogquit.psc

@@ -0,0 +1,7 @@
+Scriptname _00E_ConfirmDialogQuit Hidden
+
+function Show() global
+	if (Game.GetFormFromFile(0x163D, "Update.esm") as Message).Show() == 1
+		Game.QuitToMainMenu()
+	endif
+endfunction

source/scripts/_00e_questfunctions.psc

@@ -139,58 +139,21 @@ Function DisableDialogueQuitting() Global
 {Disables the TAB Key during dialogue. Resets automatically upon dialogue exit via Goodbye.}
 
 	if ! UI.IsMenuOpen("Dialogue Menu")
-		UnregisterDialogQuitKey()
+		EnableDialogueQuitting()
 		return
 	endif
 
-	if UI.GetBool("Dialogue Menu", "_root.DialogueMenu_mc.bEnableTab") != true
-		; Suspected non-Enderal dialoguemenu.swf replacer, rechecking value in order to be sure.
-		UI.InvokeBool("Dialogue Menu", "_root.DialogueMenu_mc.SetVariable", True)
-		if UI.GetBool("Dialogue Menu", "_root.DialogueMenu_mc.bEnableTab") != true
-			Debug.Notification("Detected incompatible dialoguemenu.swf!")
-		endif
-	endif
-
-	UI.InvokeBool("Dialogue Menu", "_root.DialogueMenu_mc.SetVariable", False)
-
-	Quest Levelsystem = Game.GetForm(0x10AA2) as Quest
-	Levelsystem.RegisterForKey(1) ; Escape
-	Levelsystem.RegisterForKey(15) ; Tab
-	int iControllerKey = Input.GetMappedKey("Tween Menu", 0x02)
-	if iControllerKey > -1
-		Levelsystem.RegisterForKey(iControllerKey) ; Controller
-	endif
-
-	Levelsystem.RegisterForMenu("Dialogue Menu")
+	EnderalFunctions.DisableDialogueQuitting()
 	
 EndFunction
 
 Function EnableDialogueQuitting() Global
 {Disables the TAB Key during dialogue. Resets automatically upon dialogue exit via Goodbye.}
 
-	UI.InvokeBool("Dialogue Menu", "_root.DialogueMenu_mc.SetVariable", True)
-	UnregisterDialogQuitKey()
+	EnderalFunctions.EnableDialogueQuitting()
 	
 EndFunction
 
-function UnregisterDialogQuitKey() Global
-	Quest Levelsystem = Game.GetForm(0x10AA2) as Quest
-	Levelsystem.UnregisterForAllKeys()
-	Levelsystem.UnregisterForMenu("Dialogue Menu")
-endfunction
-
-Event OnKeyDown(Int KeyCode)
-	if UI.IsMenuOpen("Dialogue Menu")
-		if ! UI.IsMenuOpen("MessageBoxMenu") && ! UI.GetBool("Dialogue Menu", "_root.DialogueMenu_mc.bEnableTab")
-			if (Game.GetFormFromFile(0x163D, "Update.esm") as Message).Show() == 1
-				Game.QuitToMainMenu()
-			endif
-		endif
-	else
-		UnregisterDialogQuitKey()
-	endif
-EndEvent
-
 Function RefreshFace() Global
 
 	String facegen = "bUseFaceGenPreprocessedHeads:General"
@@ -934,8 +897,6 @@ Event OnMenuClose(String MenuName)
 	If MenuName == "Book Menu"
 		MTToRemove.Remove()
 		UnregisterForMenu("Book Menu")
-	elseif MenuName == "Dialogue Menu"
-		UnregisterDialogQuitKey()
 	EndIf
 
 EndEvent

source/scripts/enderalfunctions.psc

@@ -19,6 +19,10 @@ String Function GetPlayerHash() Global Native
 ; RETURN - Returns the hexadecimal equivalent of the passed string.
 String Function StringToHex(String a_string) Global Native
 
+; Disables the TAB Key during dialogue. Resets automatically upon dialogue exit via Goodbye.
+Function DisableDialogueQuitting() Global Native
+Function EnableDialogueQuitting() Global Native
+
 bool function IsDLLLoaded() global
 	int iVer = SKSE.GetPluginVersion("EnderalSE")
 	return iVer > 0