win-capture: Fix getting proper UWP window handles

The "main" windows detected for UWP programs are basically to help
sandbox the programs -- they run in the ApplicationFrameHost process and
help reduce the possibility of other programs trying to access the
actual process window, which is a child window.

To bypass this, go through the list of child windows for the
ApplicationFrameHost window, and then find the one that's attached to
a different process; that different process will always be the target,
and will allows us to open the actual process of the UWP program.

plugins/win-capture/game-capture.c

@@ -1443,6 +1443,9 @@ static void game_capture_tick(void *data, float seconds)
 	if (activate_now) {
 		HWND hwnd = (HWND)os_atomic_load_long(&gc->hotkey_window);
 
+		if (is_uwp_window(hwnd))
+			hwnd = get_uwp_actual_window(hwnd);
+
 		if (get_window_exe(&gc->executable, hwnd)) {
 			get_window_title(&gc->title, hwnd);
 			get_window_class(&gc->class, hwnd);

plugins/win-capture/window-helpers.c

@@ -192,33 +192,92 @@ static bool check_window_valid(HWND window, enum window_search_mode mode)
 	return true;
 }
 
-static inline HWND next_window(HWND window, enum window_search_mode mode)
+bool is_uwp_window(HWND hwnd)
 {
+	wchar_t name[256];
+
+	name[0] = 0;
+	if (!GetClassNameW(hwnd, name, sizeof(name) / sizeof(wchar_t)))
+		return false;
+
+	return wcscmp(name, L"ApplicationFrameWindow") == 0;
+}
+
+HWND get_uwp_actual_window(HWND parent)
+{
+	DWORD parent_id = 0;
+	HWND child;
+
+	GetWindowThreadProcessId(parent, &parent_id);
+	child = GetWindow(parent, GW_CHILD);
+
+	while (child) {
+		DWORD child_id = 0;
+		GetWindowThreadProcessId(child, &child_id);
+
+		if (child_id != parent_id)
+			return child;
+
+		child = GetNextWindow(child, GW_HWNDNEXT);
+	}
+
+	return NULL;
+}
+
+static inline HWND next_window(HWND window, enum window_search_mode mode,
+		HWND *parent)
+{
+	if (*parent) {
+		window = *parent;
+		*parent = NULL;
+	}
+
 	while (true) {
 		window = GetNextWindow(window, GW_HWNDNEXT);
 		if (!window || check_window_valid(window, mode))
 			break;
 	}
 
+	if (is_uwp_window(window)) {
+		HWND child = get_uwp_actual_window(window);
+		if (child) {
+			*parent = window;
+			return child;
+		}
+	}
+
 	return window;
 }
 
-static inline HWND first_window(enum window_search_mode mode)
+static inline HWND first_window(enum window_search_mode mode, HWND *parent)
 {
 	HWND window = GetWindow(GetDesktopWindow(), GW_CHILD);
+
+	*parent = NULL;
+
 	if (!check_window_valid(window, mode))
-		window = next_window(window, mode);
+		window = next_window(window, mode, parent);
+
+	if (is_uwp_window(window)) {
+		HWND child = get_uwp_actual_window(window);
+		if (child) {
+			*parent = window;
+			return child;
+		}
+	}
+
 	return window;
 }
 
 void fill_window_list(obs_property_t *p, enum window_search_mode mode,
 		add_window_cb callback)
 {
-	HWND window = first_window(mode);
+	HWND parent;
+	HWND window = first_window(mode, &parent);
 
 	while (window) {
 		add_window(p, window, callback);
-		window = next_window(window, mode);
+		window = next_window(window, mode, &parent);
 	}
 }
 
@@ -268,7 +327,8 @@ HWND find_window(enum window_search_mode mode,
 		const char *title,
 		const char *exe)
 {
-	HWND window      = first_window(mode);
+	HWND parent;
+	HWND window      = first_window(mode, &parent);
 	HWND best_window = NULL;
 	int  best_rating = 0;
 
@@ -279,7 +339,7 @@ HWND find_window(enum window_search_mode mode,
 			best_window = window;
 		}
 
-		window = next_window(window, mode);
+		window = next_window(window, mode, &parent);
 	}
 
 	return best_window;

plugins/win-capture/window-helpers.h

@@ -16,6 +16,8 @@ enum window_search_mode {
 extern bool get_window_exe(struct dstr *name, HWND window);
 extern void get_window_title(struct dstr *name, HWND hwnd);
 extern void get_window_class(struct dstr *class, HWND hwnd);
+extern bool is_uwp_window(HWND hwnd);
+extern HWND get_uwp_actual_window(HWND parent);
 
 typedef bool (*add_window_cb)(const char *title, const char *class,
 		const char *exe);