obs-qsv11: Enable QSV texture-based encoding

Enables a pipeline for texture-based encoding with QSV. Utilizes OBS
NV12 output for encode to avoid offloading them from GPU, which will
increase performance. The option to select old QSV pipeline still
remains and will fallback if new pipeline fails.

plugins/obs-qsv11/CMakeLists.txt

@@ -80,6 +80,7 @@ target_link_libraries(obs-qsv11
 	d3d11
 	dxva2
 	dxgi
+	dxguid
 	)
 
 target_compile_definitions(obs-qsv11 PRIVATE DX11_D3D)

plugins/obs-qsv11/QSV_Encoder.cpp

@@ -196,6 +196,23 @@ int qsv_encoder_encode(qsv_t *pContext, uint64_t ts, uint8_t *pDataY,
 		return -1;
 }
 
+int qsv_encoder_encode_tex(qsv_t *pContext, uint64_t ts, uint32_t tex_handle,
+			   uint64_t lock_key, uint64_t *next_key,
+			   mfxBitstream **pBS)
+{
+	QSV_Encoder_Internal *pEncoder = (QSV_Encoder_Internal *)pContext;
+	mfxStatus sts = MFX_ERR_NONE;
+
+	sts = pEncoder->Encode_tex(ts, tex_handle, lock_key, next_key, pBS);
+
+	if (sts == MFX_ERR_NONE)
+		return 0;
+	else if (sts == MFX_ERR_MORE_DATA)
+		return 1;
+	else
+		return -1;
+}
+
 int qsv_encoder_close(qsv_t *pContext)
 {
 	QSV_Encoder_Internal *pEncoder = (QSV_Encoder_Internal *)pContext;

plugins/obs-qsv11/QSV_Encoder.h

@@ -131,6 +131,8 @@ void qsv_encoder_version(unsigned short *major, unsigned short *minor);
 qsv_t *qsv_encoder_open(qsv_param_t *);
 int qsv_encoder_encode(qsv_t *, uint64_t, uint8_t *, uint8_t *, uint32_t,
 		       uint32_t, mfxBitstream **pBS);
+int qsv_encoder_encode_tex(qsv_t *, uint64_t, uint32_t, uint64_t, uint64_t *,
+			   mfxBitstream **pBS);
 int qsv_encoder_headers(qsv_t *, uint8_t **pSPS, uint8_t **pPPS,
 			uint16_t *pnSPS, uint16_t *pnPPS);
 enum qsv_cpu_platform qsv_get_cpu_platform();

plugins/obs-qsv11/QSV_Encoder_Internal.cpp

@@ -569,6 +569,70 @@ mfxStatus QSV_Encoder_Internal::Encode(uint64_t ts, uint8_t *pDataY,
 	return sts;
 }
 
+mfxStatus QSV_Encoder_Internal::Encode_tex(uint64_t ts, uint32_t tex_handle,
+					   uint64_t lock_key,
+					   uint64_t *next_key,
+					   mfxBitstream **pBS)
+{
+	mfxStatus sts = MFX_ERR_NONE;
+	*pBS = NULL;
+	int nTaskIdx = GetFreeTaskIndex(m_pTaskPool, m_nTaskPool);
+	int nSurfIdx = GetFreeSurfaceIndex(m_pmfxSurfaces, m_nSurfNum);
+
+	while (MFX_ERR_NOT_FOUND == nTaskIdx || MFX_ERR_NOT_FOUND == nSurfIdx) {
+		// No more free tasks or surfaces, need to sync
+		sts = m_session.SyncOperation(
+			m_pTaskPool[m_nFirstSyncTask].syncp, 60000);
+		MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts);
+
+		mfxU8 *pTemp = m_outBitstream.Data;
+		memcpy(&m_outBitstream, &m_pTaskPool[m_nFirstSyncTask].mfxBS,
+		       sizeof(mfxBitstream));
+
+		m_pTaskPool[m_nFirstSyncTask].mfxBS.Data = pTemp;
+		m_pTaskPool[m_nFirstSyncTask].mfxBS.DataLength = 0;
+		m_pTaskPool[m_nFirstSyncTask].mfxBS.DataOffset = 0;
+		m_pTaskPool[m_nFirstSyncTask].syncp = NULL;
+		nTaskIdx = m_nFirstSyncTask;
+		m_nFirstSyncTask = (m_nFirstSyncTask + 1) % m_nTaskPool;
+		*pBS = &m_outBitstream;
+
+		nSurfIdx = GetFreeSurfaceIndex(m_pmfxSurfaces, m_nSurfNum);
+	}
+
+	mfxFrameSurface1 *pSurface = m_pmfxSurfaces[nSurfIdx];
+	//copy to default surface directly
+	pSurface->Data.TimeStamp = ts;
+	if (m_bUseD3D11 || m_bD3D9HACK) {
+		sts = simple_copytex(m_mfxAllocator.pthis, pSurface->Data.MemId,
+				     tex_handle, lock_key, next_key);
+		MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts);
+	}
+
+	for (;;) {
+		// Encode a frame asynchronously (returns immediately)
+		sts = m_pmfxENC->EncodeFrameAsync(NULL, pSurface,
+						  &m_pTaskPool[nTaskIdx].mfxBS,
+						  &m_pTaskPool[nTaskIdx].syncp);
+
+		if (MFX_ERR_NONE < sts && !m_pTaskPool[nTaskIdx].syncp) {
+			// Repeat the call if warning and no output
+			if (MFX_WRN_DEVICE_BUSY == sts)
+				MSDK_SLEEP(
+					1); // Wait if device is busy, then repeat the same call
+		} else if (MFX_ERR_NONE < sts && m_pTaskPool[nTaskIdx].syncp) {
+			sts = MFX_ERR_NONE; // Ignore warnings if output is available
+			break;
+		} else if (MFX_ERR_NOT_ENOUGH_BUFFER == sts) {
+			// Allocate more bitstream buffer memory here if needed...
+			break;
+		} else
+			break;
+	}
+
+	return sts;
+}
+
 mfxStatus QSV_Encoder_Internal::Drain()
 {
 	mfxStatus sts = MFX_ERR_NONE;

plugins/obs-qsv11/QSV_Encoder_Internal.h

@@ -70,6 +70,9 @@ public:
 	mfxStatus Encode(uint64_t ts, uint8_t *pDataY, uint8_t *pDataUV,
 			 uint32_t strideY, uint32_t strideUV,
 			 mfxBitstream **pBS);
+	mfxStatus Encode_tex(uint64_t ts, uint32_t tex_handle,
+			     uint64_t lock_key, uint64_t *next_key,
+			     mfxBitstream **pBS);
 	mfxStatus ClearData();
 	mfxStatus Reset(qsv_param_t *pParams);
 

plugins/obs-qsv11/common_directx11.cpp

@@ -57,8 +57,8 @@ IDXGIAdapter *GetIntelDeviceAdapterHandle(mfxSession session)
 		}
 	}
 
-	HRESULT hres = CreateDXGIFactory(__uuidof(IDXGIFactory2),
-					 (void **)(&g_pDXGIFactory));
+	HRESULT hres = CreateDXGIFactory1(__uuidof(IDXGIFactory2),
+					  (void **)(&g_pDXGIFactory));
 	if (FAILED(hres))
 		return NULL;
 
@@ -433,6 +433,46 @@ mfxStatus simple_unlock(mfxHDL pthis, mfxMemId mid, mfxFrameData *ptr)
 	return MFX_ERR_NONE;
 }
 
+mfxStatus simple_copytex(mfxHDL pthis, mfxMemId mid, mfxU32 tex_handle,
+			 mfxU64 lock_key, mfxU64 *next_key)
+{
+	pthis; // To suppress warning for this unused parameter
+
+	CustomMemId *memId = (CustomMemId *)mid;
+	ID3D11Texture2D *pSurface = (ID3D11Texture2D *)memId->memId;
+
+	IDXGIKeyedMutex *km;
+	ID3D11Texture2D *input_tex;
+	HRESULT hr;
+
+	hr = g_pD3D11Device->OpenSharedResource((HANDLE)(uintptr_t)tex_handle,
+						IID_ID3D11Texture2D,
+						(void **)&input_tex);
+	if (FAILED(hr)) {
+		return MFX_ERR_INVALID_HANDLE;
+	}
+
+	hr = input_tex->QueryInterface(IID_IDXGIKeyedMutex, (void **)&km);
+	if (FAILED(hr)) {
+		input_tex->Release();
+		return MFX_ERR_INVALID_HANDLE;
+	}
+
+	input_tex->SetEvictionPriority(DXGI_RESOURCE_PRIORITY_MAXIMUM);
+
+	km->AcquireSync(lock_key, INFINITE);
+
+	D3D11_TEXTURE2D_DESC desc = {0};
+	input_tex->GetDesc(&desc);
+	D3D11_BOX SrcBox = {0, 0, 0, desc.Width, desc.Height, 1};
+	g_pD3D11Ctx->CopySubresourceRegion(pSurface, 0, 0, 0, 0, input_tex, 0,
+					   &SrcBox);
+
+	km->ReleaseSync(*next_key);
+
+	return MFX_ERR_NONE;
+}
+
 mfxStatus simple_gethdl(mfxHDL pthis, mfxMemId mid, mfxHDL *handle)
 {
 	pthis; // To suppress warning for this unused parameter

plugins/obs-qsv11/common_utils.h

@@ -95,6 +95,8 @@ mfxStatus simple_lock(mfxHDL pthis, mfxMemId mid, mfxFrameData *ptr);
 mfxStatus simple_unlock(mfxHDL pthis, mfxMemId mid, mfxFrameData *ptr);
 mfxStatus simple_gethdl(mfxHDL pthis, mfxMemId mid, mfxHDL *handle);
 mfxStatus simple_free(mfxHDL pthis, mfxFrameAllocResponse *response);
+mfxStatus simple_copytex(mfxHDL pthis, mfxMemId mid, mfxU32 tex_handle,
+			 mfxU64 lock_key, mfxU64 *next_key);
 
 // =================================================================
 // Utility functions, not directly tied to Media SDK functionality

plugins/obs-qsv11/obs-qsv11-plugin-main.c

@@ -64,6 +64,7 @@ MODULE_EXPORT const char *obs_module_description(void)
 }
 
 extern struct obs_encoder_info obs_qsv_encoder;
+extern struct obs_encoder_info obs_qsv_encoder_tex;
 
 bool obs_module_load(void)
 {
@@ -76,6 +77,7 @@ bool obs_module_load(void)
 
 	if (sts == MFX_ERR_NONE) {
 		obs_register_encoder(&obs_qsv_encoder);
+		obs_register_encoder(&obs_qsv_encoder_tex);
 		MFXClose(session);
 	} else {
 		impl = MFX_IMPL_HARDWARE_ANY | MFX_IMPL_VIA_D3D9;

plugins/obs-qsv11/obs-qsv11.c

@@ -60,6 +60,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <util/platform.h>
 #include <obs-module.h>
 #include <obs-avc.h>
+#include <d3d11.h>
+#include <dxgi1_2.h>
 
 #ifndef _STDINT_H_INCLUDED
 #define _STDINT_H_INCLUDED
@@ -582,6 +584,92 @@ static void *obs_qsv_create(obs_data_t *settings, obs_encoder_t *encoder)
 	return obsqsv;
 }
 
+static HANDLE get_lib(const char *lib)
+{
+	HMODULE mod = GetModuleHandleA(lib);
+	if (mod)
+		return mod;
+
+	mod = LoadLibraryA(lib);
+	if (!mod)
+		blog(LOG_INFO, "Failed to load %s", lib);
+	return mod;
+}
+
+typedef HRESULT(WINAPI *CREATEDXGIFACTORY1PROC)(REFIID, void **);
+
+static bool is_intel_gpu_primary()
+{
+	HMODULE dxgi = get_lib("DXGI.dll");
+	CREATEDXGIFACTORY1PROC create_dxgi;
+	IDXGIFactory1 *factory;
+	IDXGIAdapter *adapter;
+	DXGI_ADAPTER_DESC desc;
+	HRESULT hr;
+
+	if (!dxgi) {
+		return false;
+	}
+	create_dxgi = (CREATEDXGIFACTORY1PROC)GetProcAddress(
+		dxgi, "CreateDXGIFactory1");
+
+	if (!create_dxgi) {
+		blog(LOG_INFO, "Failed to load D3D11/DXGI procedures");
+		return false;
+	}
+
+	hr = create_dxgi(&IID_IDXGIFactory1, &factory);
+	if (FAILED(hr)) {
+		blog(LOG_INFO, "CreateDXGIFactory1 failed");
+		return false;
+	}
+
+	hr = factory->lpVtbl->EnumAdapters(factory, 0, &adapter);
+	factory->lpVtbl->Release(factory);
+	if (FAILED(hr)) {
+		blog(LOG_INFO, "EnumAdapters failed");
+		return false;
+	}
+
+	hr = adapter->lpVtbl->GetDesc(adapter, &desc);
+	adapter->lpVtbl->Release(adapter);
+	if (FAILED(hr)) {
+		blog(LOG_INFO, "GetDesc failed");
+		return false;
+	}
+
+	/*check whether adapter 0 is Intel*/
+	if (desc.VendorId == 0x8086) {
+		return true;
+	} else {
+		return false;
+	}
+}
+
+static void *obs_qsv_create_tex(obs_data_t *settings, obs_encoder_t *encoder)
+{
+	if (!is_intel_gpu_primary()) {
+		blog(LOG_INFO,
+		     ">>> app not on intel GPU, fall back to old qsv encoder");
+		return obs_encoder_create_rerouted(encoder, "obs_qsv11_soft");
+	}
+
+	if (!obs_nv12_tex_active()) {
+		blog(LOG_INFO,
+		     ">>> nv12 tex not active, fall back to old qsv encoder");
+		return obs_encoder_create_rerouted(encoder, "obs_qsv11_soft");
+	}
+
+	if (obs_encoder_scaling_enabled(encoder)) {
+		blog(LOG_INFO,
+		     ">>> encoder scaling active, fall back to old qsv encoder");
+		return obs_encoder_create_rerouted(encoder, "obs_qsv11_soft");
+	}
+
+	blog(LOG_INFO, ">>> new qsv encoder");
+	return obs_qsv_create(settings, encoder);
+}
+
 static bool obs_qsv_extra_data(void *data, uint8_t **extra_data, size_t *size)
 {
 	struct obs_qsv *obsqsv = data;
@@ -792,8 +880,51 @@ static bool obs_qsv_encode(void *data, struct encoder_frame *frame,
 	return true;
 }
 
+static bool obs_qsv_encode_tex(void *data, uint32_t handle, int64_t pts,
+			       uint64_t lock_key, uint64_t *next_key,
+			       struct encoder_packet *packet,
+			       bool *received_packet)
+{
+	struct obs_qsv *obsqsv = data;
+
+	if (handle == GS_INVALID_HANDLE) {
+		warn("Encode failed: bad texture handle");
+		*next_key = lock_key;
+		return false;
+	}
+
+	if (!packet || !received_packet)
+		return false;
+
+	EnterCriticalSection(&g_QsvCs);
+
+	video_t *video = obs_encoder_video(obsqsv->encoder);
+	const struct video_output_info *voi = video_output_get_info(video);
+
+	mfxBitstream *pBS = NULL;
+
+	int ret;
+
+	mfxU64 qsvPTS = pts * 90000 / voi->fps_num;
+
+	ret = qsv_encoder_encode_tex(obsqsv->context, qsvPTS, handle, lock_key,
+				     next_key, &pBS);
+
+	if (ret < 0) {
+		warn("encode failed");
+		LeaveCriticalSection(&g_QsvCs);
+		return false;
+	}
+
+	parse_packet(obsqsv, packet, pBS, voi->fps_num, received_packet);
+
+	LeaveCriticalSection(&g_QsvCs);
+
+	return true;
+}
+
 struct obs_encoder_info obs_qsv_encoder = {
-	.id = "obs_qsv11",
+	.id = "obs_qsv11_soft",
 	.type = OBS_ENCODER_VIDEO,
 	.codec = "h264",
 	.get_name = obs_qsv_getname,
@@ -806,5 +937,22 @@ struct obs_encoder_info obs_qsv_encoder = {
 	.get_extra_data = obs_qsv_extra_data,
 	.get_sei_data = obs_qsv_sei,
 	.get_video_info = obs_qsv_video_info,
-	.caps = OBS_ENCODER_CAP_DYN_BITRATE,
+	.caps = OBS_ENCODER_CAP_DYN_BITRATE | OBS_ENCODER_CAP_INTERNAL,
+};
+
+struct obs_encoder_info obs_qsv_encoder_tex = {
+	.id = "obs_qsv11",
+	.type = OBS_ENCODER_VIDEO,
+	.codec = "h264",
+	.get_name = obs_qsv_getname,
+	.create = obs_qsv_create_tex,
+	.destroy = obs_qsv_destroy,
+	.caps = OBS_ENCODER_CAP_DYN_BITRATE | OBS_ENCODER_CAP_PASS_TEXTURE,
+	.encode_texture = obs_qsv_encode_tex,
+	.update = obs_qsv_update,
+	.get_properties = obs_qsv_props,
+	.get_defaults = obs_qsv_defaults,
+	.get_extra_data = obs_qsv_extra_data,
+	.get_sei_data = obs_qsv_sei,
+	.get_video_info = obs_qsv_video_info,
 };