win-capture: Fix D3D leaks on swap chain release

For game capture, hook DXGI release function to release D3D objects if
the related swap chain is also being destroyed.

An added bonus is that the game capture hook will handle swap chain
recreation for applications that don't use ResizeBuffers.

plugins/win-capture/get-graphics-offsets/dxgi-offsets.cpp

@@ -102,7 +102,8 @@ static inline void dxgi_free(dxgi_info &info)
 		DestroyWindow(info.hwnd);
 }
 
-void get_dxgi_offsets(struct dxgi_offsets *offsets)
+void get_dxgi_offsets(struct dxgi_offsets *offsets,
+		      struct dxgi_offsets2 *offsets2)
 {
 	dxgi_info info = {};
 	bool success = dxgi_init(info);
@@ -120,6 +121,8 @@ void get_dxgi_offsets(struct dxgi_offsets *offsets)
 				vtable_offset(info.module, swap1, 22);
 			swap1->Release();
 		}
+
+		offsets2->release = vtable_offset(info.module, info.swap, 2);
 	}
 
 	dxgi_free(info);

plugins/win-capture/get-graphics-offsets/get-graphics-offsets.c

@@ -8,6 +8,7 @@ int main(int argc, char *argv[])
 	struct d3d8_offsets d3d8 = {0};
 	struct d3d9_offsets d3d9 = {0};
 	struct dxgi_offsets dxgi = {0};
+	struct dxgi_offsets2 dxgi2 = {0};
 
 	WNDCLASSA wc = {0};
 	wc.style = CS_OWNDC;
@@ -24,7 +25,7 @@ int main(int argc, char *argv[])
 
 	get_d3d9_offsets(&d3d9);
 	get_d3d8_offsets(&d3d8);
-	get_dxgi_offsets(&dxgi);
+	get_dxgi_offsets(&dxgi, &dxgi2);
 
 	printf("[d3d8]\n");
 	printf("present=0x%" PRIx32 "\n", d3d8.present);
@@ -38,6 +39,7 @@ int main(int argc, char *argv[])
 	printf("present=0x%" PRIx32 "\n", dxgi.present);
 	printf("present1=0x%" PRIx32 "\n", dxgi.present1);
 	printf("resize=0x%" PRIx32 "\n", dxgi.resize);
+	printf("release=0x%" PRIx32 "\n", dxgi2.release);
 
 	(void)argc;
 	(void)argv;

plugins/win-capture/get-graphics-offsets/get-graphics-offsets.h

@@ -21,7 +21,8 @@ static inline uint32_t vtable_offset(HMODULE module, void *cls,
 	return (uint32_t)(vtable[offset] - (uintptr_t)module);
 }
 
-extern void get_dxgi_offsets(struct dxgi_offsets *offsets);
+extern void get_dxgi_offsets(struct dxgi_offsets *offsets,
+			     struct dxgi_offsets2 *offsets2);
 extern void get_d3d9_offsets(struct d3d9_offsets *offsets);
 extern void get_d3d8_offsets(struct d3d8_offsets *offsets);
 

plugins/win-capture/graphics-hook-info.h

 #pragma once
 
+#include <assert.h>
 #include <stdint.h>
 #include <stdbool.h>
 #include <string.h>
@@ -46,6 +47,10 @@ struct dxgi_offsets {
 	uint32_t present1;
 };
 
+struct dxgi_offsets2 {
+	uint32_t release;
+};
+
 struct ddraw_offsets {
 	uint32_t surface_create;
 	uint32_t surface_restore;
@@ -77,6 +82,7 @@ struct graphics_offsets {
 	struct d3d9_offsets d3d9;
 	struct dxgi_offsets dxgi;
 	struct ddraw_offsets ddraw;
+	struct dxgi_offsets2 dxgi2;
 };
 
 struct hook_info {
@@ -106,8 +112,9 @@ struct hook_info {
 	/* hook addresses */
 	struct graphics_offsets offsets;
 
-	uint32_t reserved[128];
+	uint32_t reserved[127];
 };
+static_assert(sizeof(struct hook_info) == 648, "ABI compatibility");
 
 #pragma pack(pop)
 

plugins/win-capture/graphics-hook/dxgi-capture.cpp

@@ -10,6 +10,7 @@
 #include <d3d12.h>
 #endif
 
+typedef ULONG(STDMETHODCALLTYPE *release_t)(IUnknown *);
 typedef HRESULT(STDMETHODCALLTYPE *resize_buffers_t)(IDXGISwapChain *, UINT,
 						     UINT, UINT, DXGI_FORMAT,
 						     UINT);
@@ -17,6 +18,7 @@ typedef HRESULT(STDMETHODCALLTYPE *present_t)(IDXGISwapChain *, UINT, UINT);
 typedef HRESULT(STDMETHODCALLTYPE *present1_t)(IDXGISwapChain1 *, UINT, UINT,
 					       const DXGI_PRESENT_PARAMETERS *);
 
+static struct func_hook release;
 static struct func_hook resize_buffers;
 static struct func_hook present;
 static struct func_hook present1;
@@ -80,6 +82,23 @@ static bool setup_dxgi(IDXGISwapChain *swap)
 	return false;
 }
 
+static ULONG STDMETHODCALLTYPE hook_release(IUnknown *unknown)
+{
+	unhook(&release);
+	release_t call = (release_t)release.call_addr;
+	ULONG refs = call(unknown);
+	rehook(&release);
+
+	if (unknown == data.swap && refs == 0) {
+		data.free();
+		data.swap = nullptr;
+		data.free = nullptr;
+		data.capture = nullptr;
+	}
+
+	return refs;
+}
+
 static bool resize_buffers_called = false;
 
 static HRESULT STDMETHODCALLTYPE hook_resize_buffers(IDXGISwapChain *swap,
@@ -223,6 +242,7 @@ bool hook_dxgi(void)
 	void *present_addr;
 	void *resize_addr;
 	void *present1_addr = nullptr;
+	void *release_addr = nullptr;
 
 	if (!dxgi_module) {
 		return false;
@@ -237,6 +257,9 @@ bool hook_dxgi(void)
 	if (global_hook_info->offsets.dxgi.present1)
 		present1_addr = get_offset_addr(
 			dxgi_module, global_hook_info->offsets.dxgi.present1);
+	if (global_hook_info->offsets.dxgi2.release)
+		release_addr = get_offset_addr(
+			dxgi_module, global_hook_info->offsets.dxgi2.release);
 
 	hook_init(&present, present_addr, (void *)hook_present,
 		  "IDXGISwapChain::Present");
@@ -245,11 +268,16 @@ bool hook_dxgi(void)
 	if (present1_addr)
 		hook_init(&present1, present1_addr, (void *)hook_present1,
 			  "IDXGISwapChain1::Present1");
+	if (release_addr)
+		hook_init(&release, release_addr, (void *)hook_release,
+			  "IDXGISwapChain::Release");
 
 	rehook(&resize_buffers);
 	rehook(&present);
 	if (present1_addr)
 		rehook(&present1);
+	if (release_addr)
+		rehook(&release);
 
 	hlog("Hooked DXGI");
 	return true;

plugins/win-capture/load-graphics-offsets.c

@@ -40,6 +40,8 @@ static inline bool load_offsets_from_string(struct graphics_offsets *offsets,
 		(uint32_t)config_get_uint(config, "dxgi", "present1");
 	offsets->dxgi.resize =
 		(uint32_t)config_get_uint(config, "dxgi", "resize");
+	offsets->dxgi2.release =
+		(uint32_t)config_get_uint(config, "dxgi", "release");
 
 	config_close(config);
 	return true;