From 5c2eea89bf77069e0a6a62f84de8fb6a1af62b2f Mon Sep 17 00:00:00 2001 From: praydog Date: Mon, 3 Jun 2024 13:06:11 -0700 Subject: [PATCH] D3D12Hook: Fix swapchain creation causing crashes --- src/D3D12Hook.cpp | 55 +++++++++++++++++++++++++++++++---------------- src/D3D12Hook.hpp | 7 +++--- 2 files changed, 41 insertions(+), 21 deletions(-) diff --git a/src/D3D12Hook.cpp b/src/D3D12Hook.cpp index 76e4ec51..b9a6b0a1 100644 --- a/src/D3D12Hook.cpp +++ b/src/D3D12Hook.cpp @@ -57,6 +57,35 @@ void* D3D12Hook::Streamline::link_swapchain_to_cmd_queue(void* rcx, void* rdx, v return result; } +HRESULT WINAPI D3D12Hook::create_swapchain(IDXGIFactory4* factory, IUnknown* device, HWND hwnd, const DXGI_SWAP_CHAIN_DESC* desc, const DXGI_SWAP_CHAIN_FULLSCREEN_DESC* p_fullscreen_desc, IDXGIOutput* p_restrict_to_output, IDXGISwapChain** swap_chain) { + auto create_swap_chain_fn = s_create_swapchain_hook->get_original(); + + if (g_inside_d3d12_hook) { + spdlog::info("create_swapchain (inside D3D12 hook)"); + return create_swap_chain_fn(factory, device, hwnd, desc, p_fullscreen_desc, p_restrict_to_output, swap_chain); + } + + spdlog::info("create_swapchain called"); + + std::scoped_lock _{g_framework->get_hook_monitor_mutex()}; + + bool hook_was_nullptr = g_d3d12_hook == nullptr; + + if (g_d3d12_hook != nullptr && g_framework->get_d3d12_hook() != nullptr) { + g_framework->on_reset(); // Needed to prevent a crash due to resources hanging around + g_d3d12_hook->unhook(); // Removes all vtable hooks + } + + const auto result = create_swap_chain_fn(factory, device, hwnd, desc, p_fullscreen_desc, p_restrict_to_output, swap_chain); + + // rather than waiting on the hook monitor to notice the hook isn't working + if (!hook_was_nullptr) { + g_framework->hook_d3d12(); + } + + return result; +} + void D3D12Hook::hook_streamline(HMODULE dlssg_module) try { if (D3D12Hook::s_streamline.setup) { return; @@ -346,6 +375,8 @@ bool D3D12Hook::hook() { spdlog::info("Finding command queue offset"); + m_command_queue_offset = 0; + // Find the command queue offset in the swapchain for (auto i = 0; i < 512 * sizeof(void*); i += sizeof(void*)) { const auto base = (uintptr_t)swap_chain1 + i; @@ -442,6 +473,12 @@ bool D3D12Hook::hook() { auto& present_fn = (*(void***)target_swapchain)[8]; // Present m_present_hook = std::make_unique(&present_fn, &D3D12Hook::present); + + if (s_create_swapchain_hook == nullptr) { + auto& create_swapchain_fn = (*(void***)factory)[15]; // CreateSwapChainForHwnd + s_create_swapchain_hook = std::make_unique(&create_swapchain_fn, &D3D12Hook::create_swapchain); + } + m_hooked = true; } catch (const std::exception& e) { spdlog::error("Failed to initialize hooks: {}", e.what()); @@ -786,21 +823,3 @@ HRESULT WINAPI D3D12Hook::resize_target(IDXGISwapChain3* swap_chain, const DXGI_ return result; } - -/*HRESULT WINAPI D3D12Hook::create_swap_chain(IDXGIFactory4* factory, IUnknown* device, HWND hwnd, const DXGI_SWAP_CHAIN_DESC* desc, const DXGI_SWAP_CHAIN_FULLSCREEN_DESC* p_fullscreen_desc, IDXGIOutput* p_restrict_to_output, IDXGISwapChain** swap_chain) -{ - spdlog::info("D3D12 create swapchain called"); - - auto d3d12 = g_d3d12_hook; - - d3d12->m_command_queue = (ID3D12CommandQueue*)device; - - if (d3d12->m_on_create_swap_chain) { - d3d12->m_on_create_swap_chain(*d3d12); - } - - auto create_swap_chain_fn = d3d12->m_create_swap_chain_hook->get_original(); - - return create_swap_chain_fn(factory, device, hwnd, desc, p_fullscreen_desc, p_restrict_to_output, swap_chain); -}*/ - diff --git a/src/D3D12Hook.hpp b/src/D3D12Hook.hpp index a115df46..a4b952b9 100644 --- a/src/D3D12Hook.hpp +++ b/src/D3D12Hook.hpp @@ -122,9 +122,7 @@ class D3D12Hook bool m_ignore_next_present{false}; std::unique_ptr m_present_hook{}; - //std::unique_ptr m_release_hook{}; std::unique_ptr m_swapchain_hook{}; - //std::unique_ptr m_create_swap_chain_hook{}; struct Streamline { static void* link_swapchain_to_cmd_queue(void* rcx, void* rdx, void* r8, void* r9); @@ -135,6 +133,9 @@ class D3D12Hook }; static inline Streamline s_streamline{}; + + // This is static because unhooking it seems to cause a crash sometimes + static inline std::unique_ptr s_create_swapchain_hook{}; OnPresentFn m_on_present{ nullptr }; OnPresentFn m_on_post_present{ nullptr }; @@ -145,6 +146,6 @@ class D3D12Hook static HRESULT WINAPI present(IDXGISwapChain3* swap_chain, uint64_t sync_interval, uint64_t flags, void* r9); static HRESULT WINAPI resize_buffers(IDXGISwapChain3* swap_chain, UINT buffer_count, UINT width, UINT height, DXGI_FORMAT new_format, UINT swap_chain_flags); static HRESULT WINAPI resize_target(IDXGISwapChain3* swap_chain, const DXGI_MODE_DESC* new_target_parameters); - //static HRESULT WINAPI create_swap_chain(IDXGIFactory4* factory, IUnknown* device, HWND hwnd, const DXGI_SWAP_CHAIN_DESC* desc, const DXGI_SWAP_CHAIN_FULLSCREEN_DESC* p_fullscreen_desc, IDXGIOutput* p_restrict_to_output, IDXGISwapChain** swap_chain); + static HRESULT WINAPI create_swapchain(IDXGIFactory4* factory, IUnknown* device, HWND hwnd, const DXGI_SWAP_CHAIN_DESC* desc, const DXGI_SWAP_CHAIN_FULLSCREEN_DESC* p_fullscreen_desc, IDXGIOutput* p_restrict_to_output, IDXGISwapChain** swap_chain); };