UI: Warn user if multiple instances of the UI are open

Uses a named mutex to detect if multiple instances of the program are
open, and if so warns the user.  When running in portable mode, uses a
separate unique mutex name mapped to the user's config directory to
ensure that no two portable builds use the same config directory.  This
way, portable builds do not conflict with normal builds or other
separate portable builds.

UI/data/locale/en-US.ini

@@ -72,6 +72,11 @@ RemuxRecordings="Remux Recordings"
 Next="Next"
 Back="Back"
 
+# warning if program already open
+AlreadyRunning.Title="OBS is already running"
+AlreadyRunning.Text="OBS is already running!  Unless you meant to do this, please shut down any existing instances of OBS before trying to run a new instance.  If you have OBS set to minimize to the system tray, please check to see if it's still running there."
+AlreadyRunning.LaunchAnyway="Launch Anyway"
+
 # copy filters
 Copy.Filters="Copy Filters"
 Paste.Filters="Paste Filters"

UI/obs-app.cpp

@@ -61,7 +61,7 @@ static log_handler_t def_log_handler;
 static string currentLogFile;
 static string lastLogFile;
 
-static bool portable_mode = false;
+bool portable_mode = false;
 static bool log_verbose = false;
 static bool unfiltered_log = false;
 bool opt_start_streaming = false;
@@ -1327,6 +1327,46 @@ static int run_program(fstream &logFile, int argc, char *argv[])
 
 		program.installTranslator(&translator);
 
+#ifdef _WIN32
+		/* --------------------------------------- */
+		/* check and warn if already running       */
+
+		bool already_running = false;
+		RunOnceMutex rom = GetRunOnceMutex(already_running);
+
+		if (already_running) {
+			blog(LOG_WARNING, "\n================================");
+			blog(LOG_WARNING, "Warning: OBS is already running!");
+			blog(LOG_WARNING, "================================\n");
+
+			QMessageBox::StandardButtons buttons(
+					QMessageBox::Yes | QMessageBox::Cancel);
+			QMessageBox mb(QMessageBox::Question,
+					QTStr("AlreadyRunning.Title"),
+					QTStr("AlreadyRunning.Text"),
+					buttons,
+					nullptr);
+			mb.setButtonText(QMessageBox::Yes,
+					QTStr("AlreadyRunning.LaunchAnyway"));
+			mb.setButtonText(QMessageBox::Cancel, QTStr("Cancel"));
+			mb.setDefaultButton(QMessageBox::Cancel);
+
+			QMessageBox::StandardButton button;
+			button = (QMessageBox::StandardButton)mb.exec();
+			if (button == QMessageBox::Cancel) {
+				blog(LOG_INFO, "User shut down the program "
+						"because OBS was already "
+						"running");
+				return 0;
+			}
+
+			blog(LOG_WARNING, "User is now running a secondary "
+					"instance of OBS!");
+		}
+
+		/* --------------------------------------- */
+#endif
+
 		if (!program.OBSInit())
 			return 0;
 

UI/obs-app.hpp

@@ -176,6 +176,7 @@ static inline int GetProfilePath(char *path, size_t size, const char *file)
 	return window->GetProfilePath(path, size, file);
 }
 
+extern bool portable_mode;
 extern bool opt_start_streaming;
 extern bool opt_start_recording;
 extern bool opt_start_replaybuffer;

UI/platform-windows.cpp

@@ -34,6 +34,7 @@ using namespace std;
 #include <mmdeviceapi.h>
 #include <audiopolicy.h>
 
+#include <util/windows/WinHandle.hpp>
 #include <util/windows/HRError.hpp>
 #include <util/windows/ComPtr.hpp>
 
@@ -271,3 +272,65 @@ uint64_t CurrentMemoryUsage()
 
 	return (uint64_t)pmc.WorkingSetSize;
 }
+
+struct RunOnceMutexData {
+	WinHandle handle;
+
+	inline RunOnceMutexData(HANDLE h) : handle(h) {}
+};
+
+RunOnceMutex::RunOnceMutex(RunOnceMutex &&rom)
+{
+	delete data;
+	data = rom.data;
+	rom.data = nullptr;
+}
+
+RunOnceMutex::~RunOnceMutex()
+{
+	delete data;
+}
+
+RunOnceMutex &RunOnceMutex::operator=(RunOnceMutex &&rom)
+{
+	delete data;
+	data = rom.data;
+	rom.data = nullptr;
+	return *this;
+}
+
+RunOnceMutex GetRunOnceMutex(bool &already_running)
+{
+	string name;
+
+	if (!portable_mode) {
+		name = "OBSStudioCore";
+	} else {
+		char path[500];
+		*path = 0;
+		GetConfigPath(path, sizeof(path), "");
+		name = "OBSStudioPortable";
+		name += path;
+	}
+
+	BPtr<wchar_t> wname;
+	os_utf8_to_wcs_ptr(name.c_str(), name.size(), &wname);
+
+	if (wname) {
+		wchar_t *temp = wname;
+		while (*temp) {
+			if (!iswalnum(*temp))
+				*temp = L'_';
+			temp++;
+		}
+	}
+
+	HANDLE h = OpenMutexW(SYNCHRONIZE, false, wname.Get());
+	already_running = !!h;
+
+	if (!already_running)
+		h = CreateMutexW(nullptr, false, wname.Get());
+
+	RunOnceMutex rom(h ? new RunOnceMutexData(h) : nullptr);
+	return rom;
+}

UI/platform.hpp

@@ -44,6 +44,22 @@ void SetProcessPriority(const char *priority);
 void SetWin32DropStyle(QWidget *window);
 bool DisableAudioDucking(bool disable);
 uint64_t CurrentMemoryUsage();
+
+struct RunOnceMutexData;
+
+class RunOnceMutex {
+	RunOnceMutexData *data = nullptr;
+public:
+	RunOnceMutex(RunOnceMutexData *data_) : data(data_) {}
+	RunOnceMutex(const RunOnceMutex &rom) = delete;
+	RunOnceMutex(RunOnceMutex &&rom);
+	~RunOnceMutex();
+
+	RunOnceMutex &operator=(const RunOnceMutex &rom) = delete;
+	RunOnceMutex &operator=(RunOnceMutex &&rom);
+};
+
+RunOnceMutex GetRunOnceMutex(bool &already_running);
 #endif
 
 #ifdef __APPLE__