win-capture: Fall back to creating d3d contexts if offsets bad

plugins/win-capture/graphics-hook/d3d8-capture.cpp

@@ -289,17 +289,90 @@ static HRESULT STDMETHODCALLTYPE hook_present(IDirect3DDevice8 *device,
 	return hr;
 }
 
+typedef IDirect3D8 *(WINAPI *d3d8create_t)(UINT);
+
+static bool manually_get_d3d8_present_addr(HMODULE d3d8_module,
+		void **present_addr)
+{
+	d3d8create_t create;
+	D3DPRESENT_PARAMETERS pp;
+	HRESULT hr;
+
+	IDirect3DDevice8 *device;
+	IDirect3D8 *d3d8;
+
+	hlog("D3D8 value invalid, manually obtaining");
+
+	create = (d3d8create_t)GetProcAddress(d3d8_module, "Direct3DCreate8");
+	if (!create) {
+		hlog("Failed to load Direct3DCreate8");
+		return false;
+	}
+
+	d3d8 = create(D3D_SDK_VERSION);
+	if (!d3d8) {
+		hlog("Failed to create D3D8 context");
+		return false;
+	}
+
+	memset(&pp, 0, sizeof(pp));
+	pp.Windowed                 = true;
+	pp.SwapEffect               = D3DSWAPEFFECT_FLIP;
+	pp.BackBufferFormat         = D3DFMT_A8R8G8B8;
+	pp.BackBufferWidth          = 2;
+	pp.BackBufferHeight         = 2;
+	pp.BackBufferCount          = 1;
+	pp.hDeviceWindow            = dummy_window;
+
+	hr = d3d8->CreateDevice(D3DADAPTER_DEFAULT, D3DDEVTYPE_HAL,
+			dummy_window, D3DCREATE_HARDWARE_VERTEXPROCESSING, &pp,
+			&device);
+	d3d8->Release();
+
+	if (SUCCEEDED(hr)) {
+		uintptr_t *vtable = *(uintptr_t**)device;
+		*present_addr = (void*)vtable[15];
+
+		device->Release();
+	} else {
+		hlog("Failed to create D3D8 device");
+		return false;
+	}
+
+	return true;
+}
+
 bool hook_d3d8(void)
 {
 	HMODULE d3d8_module = get_system_module("d3d8.dll");
-	void *present_addr;
+	uint32_t d3d8_size;
+	void *present_addr = nullptr;
 
 	if (!d3d8_module) {
 		return false;
 	}
 
-	present_addr = get_offset_addr(d3d8_module,
-			global_hook_info->offsets.d3d8.present);
+	d3d8_size = module_size(d3d8_module);
+
+	if (global_hook_info->offsets.d3d8.present < d3d8_size) {
+		present_addr = get_offset_addr(d3d8_module,
+				global_hook_info->offsets.d3d8.present);
+	} else {
+		if (!dummy_window) {
+			return false;
+		}
+
+		if (!manually_get_d3d8_present_addr(d3d8_module,
+					&present_addr)) {
+			hlog("Failed to get D3D8 value");
+			return true;
+		}
+	}
+
+	if (!present_addr) {
+		hlog("Invalid D3D8 value");
+		return true;
+	}
 
 	hook_init(&present, present_addr, (void*)hook_present,
 			"IDirect3DDevice8::Present");
@@ -307,6 +380,5 @@ bool hook_d3d8(void)
 	rehook(&present);
 
 	hlog("Hooked D3D8");
-
 	return true;
 }

plugins/win-capture/graphics-hook/d3d9-capture.cpp

@@ -785,34 +785,132 @@ static void setup_reset_hooks(IDirect3DDevice9 *device)
 	hooked_reset = true;
 }
 
+typedef HRESULT (WINAPI *d3d9create_ex_t)(UINT, IDirect3D9Ex**);
+
+static bool manually_get_d3d9_addrs(HMODULE d3d9_module,
+		void **present_addr,
+		void **present_ex_addr,
+		void **present_swap_addr)
+{
+	d3d9create_ex_t create_ex;
+	D3DPRESENT_PARAMETERS pp;
+	HRESULT hr;
+
+	IDirect3DDevice9Ex *device;
+	IDirect3D9Ex *d3d9ex;
+
+	hlog("D3D9 values invalid, manually obtaining");
+
+	create_ex = (d3d9create_ex_t)GetProcAddress(d3d9_module,
+			"Direct3DCreate9Ex");
+	if (!create_ex) {
+		hlog("Failed to load Direct3DCreate9Ex");
+		return false;
+	}
+	if (FAILED(create_ex(D3D_SDK_VERSION, &d3d9ex))) {
+		hlog("Failed to create D3D9 context");
+		return false;
+	}
+
+	memset(&pp, 0, sizeof(pp));
+	pp.Windowed                 = 1;
+	pp.SwapEffect               = D3DSWAPEFFECT_FLIP;
+	pp.BackBufferFormat         = D3DFMT_A8R8G8B8;
+	pp.BackBufferCount          = 1;
+	pp.hDeviceWindow            = (HWND)dummy_window;
+	pp.PresentationInterval     = D3DPRESENT_INTERVAL_IMMEDIATE;
+
+	hr = d3d9ex->CreateDeviceEx(D3DADAPTER_DEFAULT, D3DDEVTYPE_HAL,
+			dummy_window,
+			D3DCREATE_HARDWARE_VERTEXPROCESSING |
+			D3DCREATE_NOWINDOWCHANGES, &pp, NULL, &device);
+	d3d9ex->Release();
+
+	if (SUCCEEDED(hr)) {
+		uintptr_t *vtable = *(uintptr_t**)device;
+		IDirect3DSwapChain9 *swap;
+
+		*present_addr = (void*)vtable[17];
+		*present_ex_addr = (void*)vtable[121];
+
+		hr = device->GetSwapChain(0, &swap);
+		if (SUCCEEDED(hr)) {
+			vtable = *(uintptr_t**)swap;
+			*present_swap_addr = (void*)vtable[3];
+
+			swap->Release();
+		}
+
+		device->Release();
+	} else {
+		hlog("Failed to create D3D9 device");
+		return false;
+	}
+
+	return true;
+}
+
 bool hook_d3d9(void)
 {
 	HMODULE d3d9_module = get_system_module("d3d9.dll");
-	void *present_addr;
-	void *present_ex_addr;
-	void *present_swap_addr;
+	uint32_t d3d9_size;
+	void *present_addr = nullptr;
+	void *present_ex_addr = nullptr;
+	void *present_swap_addr = nullptr;
 
 	if (!d3d9_module) {
 		return false;
 	}
 
-	present_addr = get_offset_addr(d3d9_module,
-			global_hook_info->offsets.d3d9.present);
-	present_ex_addr = get_offset_addr(d3d9_module,
-			global_hook_info->offsets.d3d9.present_ex);
-	present_swap_addr = get_offset_addr(d3d9_module,
-			global_hook_info->offsets.d3d9.present_swap);
+	d3d9_size = module_size(d3d9_module);
 
-	hook_init(&present, present_addr, (void*)hook_present,
-			"IDirect3DDevice9::Present");
-	hook_init(&present_ex, present_ex_addr, (void*)hook_present_ex,
-			"IDirect3DDevice9Ex::PresentEx");
-	hook_init(&present_swap, present_swap_addr, (void*)hook_present_swap,
-			"IDirect3DSwapChain9::Present");
+	if (global_hook_info->offsets.d3d9.present      < d3d9_size &&
+	    global_hook_info->offsets.d3d9.present_ex   < d3d9_size &&
+	    global_hook_info->offsets.d3d9.present_swap < d3d9_size) {
 
-	rehook(&present_swap);
-	rehook(&present_ex);
-	rehook(&present);
+		present_addr = get_offset_addr(d3d9_module,
+				global_hook_info->offsets.d3d9.present);
+		present_ex_addr = get_offset_addr(d3d9_module,
+				global_hook_info->offsets.d3d9.present_ex);
+		present_swap_addr = get_offset_addr(d3d9_module,
+				global_hook_info->offsets.d3d9.present_swap);
+	} else {
+		if (!dummy_window) {
+			return false;
+		}
+
+		if (!manually_get_d3d9_addrs(d3d9_module,
+					&present_addr,
+					&present_ex_addr,
+					&present_swap_addr)) {
+			hlog("Failed to get D3D9 values");
+			return true;
+		}
+	}
+
+	if (!present_addr && !present_ex_addr && !present_swap_addr) {
+		hlog("Invalid D3D9 values");
+		return true;
+	}
+
+	if (present_swap_addr) {
+		hook_init(&present_swap, present_swap_addr,
+				(void*)hook_present_swap,
+				"IDirect3DSwapChain9::Present");
+		rehook(&present_swap);
+	}
+	if (present_ex_addr) {
+		hook_init(&present_ex, present_ex_addr,
+				(void*)hook_present_ex,
+				"IDirect3DDevice9Ex::PresentEx");
+		rehook(&present_ex);
+	}
+	if (present_addr) {
+		hook_init(&present, present_addr,
+				(void*)hook_present,
+				"IDirect3DDevice9::Present");
+		rehook(&present);
+	}
 
 	hlog("Hooked D3D9");
 	return true;

plugins/win-capture/graphics-hook/graphics-hook.c

@@ -35,11 +35,13 @@ HANDLE                         signal_exit                     = NULL;
 HANDLE                         tex_mutexes[2]                  = {NULL, NULL};
 static HANDLE                  filemap_hook_info               = NULL;
 
+static HINSTANCE               dll_inst                        = NULL;
 static volatile bool           stop_loop                       = false;
 static HANDLE                  capture_thread                  = NULL;
 char                           system_path[MAX_PATH]           = {0};
 char                           process_name[MAX_PATH]          = {0};
 char                           keepalive_name[64]              = {0};
+HWND                           dummy_window                    = NULL;
 
 static unsigned int            shmem_id_counter                = 0;
 static void                    *shmem_info                     = NULL;
@@ -50,6 +52,7 @@ static struct thread_data      thread_data                     = {0};
 volatile bool                  active                          = false;
 struct hook_info               *global_hook_info               = NULL;
 
+
 static inline void wait_for_dll_main_finish(HANDLE thread_handle)
 {
 	if (thread_handle) {
@@ -179,6 +182,55 @@ static inline bool init_hook_info(void)
 	return true;
 }
 
+#define DEF_FLAGS (WS_POPUP | WS_CLIPCHILDREN | WS_CLIPSIBLINGS)
+
+static DWORD WINAPI dummy_window_thread(LPVOID *unused)
+{
+	static const wchar_t dummy_window_class[] = L"temp_d3d_window_4039785";
+	WNDCLASSW wc;
+	MSG msg;
+
+	memset(&wc, 0, sizeof(wc));
+	wc.style = CS_OWNDC;
+	wc.hInstance = dll_inst;
+	wc.lpfnWndProc = (WNDPROC)DefWindowProc;
+	wc.lpszClassName = dummy_window_class;
+
+	if (!RegisterClass(&wc)) {
+		hlog("Failed to create temp D3D window class: %lu",
+				GetLastError());
+		return 0;
+	}
+
+	dummy_window = CreateWindowExW(0, dummy_window_class, L"Temp Window",
+			DEF_FLAGS, 0, 0, 1, 1, NULL, NULL, dll_inst, NULL);
+	if (!dummy_window) {
+		hlog("Failed to create temp D3D window: %lu", GetLastError());
+		return 0;
+	}
+
+	while (GetMessage(&msg, NULL, 0, 0)) {
+		TranslateMessage(&msg);
+		DispatchMessage(&msg);
+	}
+
+	(void)unused;
+	return 0;
+}
+
+static inline void init_dummy_window_thread(void)
+{
+	HANDLE thread = CreateThread(NULL, 0, dummy_window_thread, NULL, 0,
+			NULL);
+	if (!thread) {
+		hlog("Failed to create temp D3D window thread: %lu",
+				GetLastError());
+		return;
+	}
+
+	CloseHandle(thread);
+}
+
 static inline bool init_hook(HANDLE thread_handle)
 {
 	wait_for_dll_main_finish(thread_handle);
@@ -202,6 +254,7 @@ static inline bool init_hook(HANDLE thread_handle)
 		return false;
 	}
 
+	init_dummy_window_thread();
 	log_current_process();
 
 	SetEvent(signal_restart);
@@ -737,6 +790,8 @@ BOOL WINAPI DllMain(HINSTANCE hinst, DWORD reason, LPVOID unused1)
 	if (reason == DLL_PROCESS_ATTACH) {
 		wchar_t name[MAX_PATH];
 
+		dll_inst = hinst;
+
 		HANDLE cur_thread = OpenThread(THREAD_ALL_ACCESS, false,
 				GetCurrentThreadId());
 

plugins/win-capture/graphics-hook/graphics-hook.h

@@ -7,6 +7,7 @@
 
 #include "../graphics-hook-info.h"
 #include <ipc-util/pipe.h>
+#include <psapi.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -91,6 +92,7 @@ extern HANDLE tex_mutexes[2];
 extern char system_path[MAX_PATH];
 extern char process_name[MAX_PATH];
 extern char keepalive_name[64];
+extern HWND dummy_window;
 extern volatile bool active;
 
 static inline const char *get_process_name(void)
@@ -108,6 +110,14 @@ static inline HMODULE get_system_module(const char *module)
 	return GetModuleHandleA(base_path);
 }
 
+static inline uint32_t module_size(HMODULE module)
+{
+	MODULEINFO info;
+	bool success = !!GetModuleInformation(GetCurrentProcess(), module,
+			&info, sizeof(info));
+	return success ? info.SizeOfImage : 0;
+}
+
 static inline HMODULE load_system_library(const char *name)
 {
 	char base_path[MAX_PATH];