win-capture: Hook Reset/ResetEx in Present* funcs

The virtual address table values for Reset/ResetEx can sometimes point
to functions that are in libraries outside of D3D8.dll and D3D9.dll, and
will cause a crash if used.  Instead, just hook Reset/ResetEx when one
of the Present* functions are called.

plugins/win-capture/get-graphics-offsets/d3d8-offsets.cpp

@@ -76,7 +76,6 @@ void get_d3d8_offsets(struct d3d8_offsets *offsets)
 
 	if (success) {
 		offsets->present = vtable_offset(info.module, info.device, 15);
-		offsets->reset   = vtable_offset(info.module, info.device, 14);
 	}
 
 	d3d8_free(info);

plugins/win-capture/get-graphics-offsets/d3d9-offsets.cpp

@@ -87,11 +87,8 @@ void get_d3d9_offsets(struct d3d9_offsets *offsets)
 
 	if (success) {
 		offsets->present = vtable_offset(info.module, info.device, 17);
-		offsets->reset = vtable_offset(info.module, info.device, 16);
 		offsets->present_ex = vtable_offset(info.module, info.device,
 				121);
-		offsets->reset_ex = vtable_offset(info.module, info.device,
-				132);
 		offsets->present_swap = vtable_offset(info.module, info.swap,
 				3);
 	}

plugins/win-capture/get-graphics-offsets/get-graphics-offsets.c

@@ -27,13 +27,10 @@ int main(int argc, char *argv[])
 
 	printf("[d3d8]\n");
 	printf("present=0x%"PRIx32"\n", d3d8.present);
-	printf("reset=0x%"PRIx32"\n", d3d8.reset);
 	printf("[d3d9]\n");
 	printf("present=0x%"PRIx32"\n", d3d9.present);
 	printf("present_ex=0x%"PRIx32"\n", d3d9.present_ex);
 	printf("present_swap=0x%"PRIx32"\n", d3d9.present_swap);
-	printf("reset=0x%"PRIx32"\n", d3d9.reset);
-	printf("reset_ex=0x%"PRIx32"\n", d3d9.reset_ex);
 	printf("[dxgi]\n");
 	printf("present=0x%"PRIx32"\n", dxgi.present);
 	printf("resize=0x%"PRIx32"\n", dxgi.resize);

plugins/win-capture/graphics-hook-info.h

@@ -27,15 +27,12 @@
 
 struct d3d8_offsets {
 	uint32_t present;
-	uint32_t reset;
 };
 
 struct d3d9_offsets {
 	uint32_t present;
 	uint32_t present_ex;
 	uint32_t present_swap;
-	uint32_t reset;
-	uint32_t reset_ex;
 };
 
 struct dxgi_offsets {

plugins/win-capture/graphics-hook/d3d8-capture.cpp

@@ -252,6 +252,19 @@ static HRESULT STDMETHODCALLTYPE hook_reset(IDirect3DDevice8 *device,
 	return hr;
 }
 
+static bool hooked_reset = false;
+
+static void setup_reset_hooks(IDirect3DDevice8 *device)
+{
+	uintptr_t *vtable = *(uintptr_t**)device;
+
+	hook_init(&reset, (void*)vtable[14], (void*)hook_reset,
+			"IDirect3DDevice8::Reset");
+	rehook(&reset);
+
+	hooked_reset = true;
+}
+
 static HRESULT STDMETHODCALLTYPE hook_present(IDirect3DDevice8 *device,
 		CONST RECT *src_rect, CONST RECT *dst_rect,
 		HWND override_window, CONST RGNDATA *dirty_region)
@@ -259,6 +272,9 @@ static HRESULT STDMETHODCALLTYPE hook_present(IDirect3DDevice8 *device,
 	IDirect3DSurface8 *backbuffer;
 	HRESULT hr;
 
+	if (!hooked_reset)
+		setup_reset_hooks(device);
+
 	backbuffer = d3d8_get_backbuffer(device);
 	if (backbuffer) {
 		d3d8_capture(device, backbuffer);
@@ -277,7 +293,6 @@ bool hook_d3d8(void)
 {
 	HMODULE d3d8_module = get_system_module("d3d8.dll");
 	void *present_addr;
-	void *reset_addr;
 
 	if (!d3d8_module) {
 		return false;
@@ -285,16 +300,11 @@ bool hook_d3d8(void)
 
 	present_addr = get_offset_addr(d3d8_module,
 			global_hook_info->offsets.d3d8.present);
-	reset_addr = get_offset_addr(d3d8_module,
-			global_hook_info->offsets.d3d8.reset);
 
 	hook_init(&present, present_addr, (void*)hook_present,
 			"IDirect3DDevice8::Present");
-	hook_init(&reset, reset_addr, (void*)hook_reset,
-			"IDirect3DDevice8::Reset");
 
 	rehook(&present);
-	rehook(&reset);
 
 	hlog("Hooked D3D8");
 

plugins/win-capture/graphics-hook/d3d9-capture.cpp

@@ -648,6 +648,9 @@ static inline void present_end(IDirect3DDevice9 *device,
 	}
 }
 
+static bool hooked_reset = false;
+static void setup_reset_hooks(IDirect3DDevice9 *device);
+
 static HRESULT STDMETHODCALLTYPE hook_present(IDirect3DDevice9 *device,
 		CONST RECT *src_rect, CONST RECT *dst_rect,
 		HWND override_window, CONST RGNDATA *dirty_region)
@@ -655,6 +658,9 @@ static HRESULT STDMETHODCALLTYPE hook_present(IDirect3DDevice9 *device,
 	IDirect3DSurface9 *backbuffer = nullptr;
 	HRESULT hr;
 
+	if (!hooked_reset)
+		setup_reset_hooks(device);
+
 	present_begin(device, backbuffer);
 
 	unhook(&present);
@@ -674,6 +680,9 @@ static HRESULT STDMETHODCALLTYPE hook_present_ex(IDirect3DDevice9 *device,
 	IDirect3DSurface9 *backbuffer = nullptr;
 	HRESULT hr;
 
+	if (!hooked_reset)
+		setup_reset_hooks(device);
+
 	present_begin(device, backbuffer);
 
 	unhook(&present_ex);
@@ -702,8 +711,12 @@ static HRESULT STDMETHODCALLTYPE hook_present_swap(IDirect3DSwapChain9 *swap,
 		}
 	}
 
-	if (device)
+	if (device) {
+		if (!hooked_reset)
+			setup_reset_hooks(device);
+
 		present_begin(device, backbuffer);
+	}
 
 	unhook(&present_swap);
 	present_swap_t call = (present_swap_t)present_swap.call_addr;
@@ -749,14 +762,35 @@ static HRESULT STDMETHODCALLTYPE hook_reset_ex(IDirect3DDevice9 *device,
 	return hr;
 }
 
+static void setup_reset_hooks(IDirect3DDevice9 *device)
+{
+	IDirect3DDevice9Ex *d3d9ex = nullptr;
+	uintptr_t *vtable = *(uintptr_t**)device;
+	HRESULT hr;
+
+	hook_init(&reset, (void*)vtable[16], (void*)hook_reset,
+			"IDirect3DDevice9::Reset");
+	rehook(&reset);
+
+	hr = device->QueryInterface(__uuidof(IDirect3DDevice9Ex),
+			(void**)&d3d9ex);
+	if (SUCCEEDED(hr)) {
+		hook_init(&reset_ex, (void*)vtable[132], (void*)hook_reset_ex,
+				"IDirect3DDevice9Ex::ResetEx");
+		rehook(&reset_ex);
+
+		d3d9ex->Release();
+	}
+
+	hooked_reset = true;
+}
+
 bool hook_d3d9(void)
 {
 	HMODULE d3d9_module = get_system_module("d3d9.dll");
 	void *present_addr;
 	void *present_ex_addr;
 	void *present_swap_addr;
-	void *reset_addr;
-	void *reset_ex_addr;
 
 	if (!d3d9_module) {
 		return false;
@@ -768,10 +802,6 @@ bool hook_d3d9(void)
 			global_hook_info->offsets.d3d9.present_ex);
 	present_swap_addr = get_offset_addr(d3d9_module,
 			global_hook_info->offsets.d3d9.present_swap);
-	reset_addr = get_offset_addr(d3d9_module,
-			global_hook_info->offsets.d3d9.reset);
-	reset_ex_addr = get_offset_addr(d3d9_module,
-			global_hook_info->offsets.d3d9.reset_ex);
 
 	hook_init(&present, present_addr, (void*)hook_present,
 			"IDirect3DDevice9::Present");
@@ -779,13 +809,7 @@ bool hook_d3d9(void)
 			"IDirect3DDevice9Ex::PresentEx");
 	hook_init(&present_swap, present_swap_addr, (void*)hook_present_swap,
 			"IDirect3DSwapChain9::Present");
-	hook_init(&reset, reset_addr, (void*)hook_reset,
-			"IDirect3DDevice9::Reset");
-	hook_init(&reset_ex, reset_ex_addr, (void*)hook_reset_ex,
-			"IDirect3DDevice9Ex::ResetEx");
 
-	rehook(&reset_ex);
-	rehook(&reset);
 	rehook(&present_swap);
 	rehook(&present_ex);
 	rehook(&present);

plugins/win-capture/graphics-hook/graphics-hook.c

@@ -238,8 +238,7 @@ static void free_hook(void)
 
 static inline bool d3d8_hookable(void)
 {
-	return !!global_hook_info->offsets.d3d8.present &&
-		!!global_hook_info->offsets.d3d8.reset;
+	return !!global_hook_info->offsets.d3d8.present;
 }
 
 static inline bool ddraw_hookable(void)
@@ -258,9 +257,7 @@ static inline bool d3d9_hookable(void)
 {
 	return !!global_hook_info->offsets.d3d9.present &&
 		!!global_hook_info->offsets.d3d9.present_ex &&
-		!!global_hook_info->offsets.d3d9.present_swap &&
-		!!global_hook_info->offsets.d3d9.reset &&
-		!!global_hook_info->offsets.d3d9.reset_ex;
+		!!global_hook_info->offsets.d3d9.present_swap;
 }
 
 static inline bool dxgi_hookable(void)

plugins/win-capture/load-graphics-offsets.c

@@ -23,8 +23,6 @@ static inline bool load_offsets_from_string(struct graphics_offsets *offsets,
 
 	offsets->d3d8.present =
 		(uint32_t)config_get_uint(config, "d3d8", "present");
-	offsets->d3d8.reset =
-		(uint32_t)config_get_uint(config, "d3d8", "reset");
 
 	offsets->d3d9.present =
 		(uint32_t)config_get_uint(config, "d3d9", "present");
@@ -32,10 +30,6 @@ static inline bool load_offsets_from_string(struct graphics_offsets *offsets,
 		(uint32_t)config_get_uint(config, "d3d9", "present_ex");
 	offsets->d3d9.present_swap =
 		(uint32_t)config_get_uint(config, "d3d9", "present_swap");
-	offsets->d3d9.reset =
-		(uint32_t)config_get_uint(config, "d3d9", "reset");
-	offsets->d3d9.reset_ex =
-		(uint32_t)config_get_uint(config, "d3d9", "reset_ex");
 
 	offsets->dxgi.present =
 		(uint32_t)config_get_uint(config, "dxgi", "present");