libobs: Allow filter processing function to return false

(Note: this commit also modifies the obs-filters and test-input modules)

Changes the obs_source_process_filter_begin return type so that it
returns true/false to indicate that filter processing should or should
not continue (for example if the filter is bypassed or if there's some
other sort of issue that causes the filtering to fail)

libobs/obs-source.c

@@ -2660,7 +2660,7 @@ static inline bool can_bypass(obs_source_t *target, obs_source_t *parent,
 		((parent_flags & OBS_SOURCE_ASYNC) == 0);
 }
 
-void obs_source_process_filter_begin(obs_source_t *filter,
+bool obs_source_process_filter_begin(obs_source_t *filter,
 		enum gs_color_format format,
 		enum obs_allow_direct_render allow_direct)
 {
@@ -2669,7 +2669,7 @@ void obs_source_process_filter_begin(obs_source_t *filter,
 	int          cx, cy;
 
 	if (!obs_ptr_valid(filter, "obs_source_process_filter_begin"))
-		return;
+		return false;
 
 	target       = obs_filter_get_target(filter);
 	parent       = obs_filter_get_parent(filter);
@@ -2677,12 +2677,12 @@ void obs_source_process_filter_begin(obs_source_t *filter,
 	if (!target) {
 		blog(LOG_INFO, "filter '%s' being processed with no target!",
 				filter->context.name);
-		return;
+		return false;
 	}
 	if (!parent) {
 		blog(LOG_INFO, "filter '%s' being processed with no parent!",
 				filter->context.name);
-		return;
+		return false;
 	}
 
 	target_flags = target->info.output_flags;
@@ -2697,12 +2697,12 @@ void obs_source_process_filter_begin(obs_source_t *filter,
 	 * using the filter effect instead of rendering to texture to reduce
 	 * the total number of passes */
 	if (can_bypass(target, parent, parent_flags, allow_direct)) {
-		return;
+		return true;
 	}
 
 	if (!cx || !cy) {
 		obs_source_skip_video_filter(filter);
-		return;
+		return false;
 	}
 
 	if (!filter->filter_texrender)
@@ -2730,6 +2730,7 @@ void obs_source_process_filter_begin(obs_source_t *filter,
 	}
 
 	gs_blend_state_pop();
+	return true;
 }
 
 void obs_source_process_filter_tech_end(obs_source_t *filter, gs_effect_t *effect,

libobs/obs.h

@@ -968,8 +968,11 @@ EXPORT void obs_source_release_frame(obs_source_t *source,
  *
  * After calling this, set your parameters for the effect, then call
  * obs_source_process_filter_end to draw the filter.
+ *
+ * Returns true if filtering should continue, false if the filter is bypassed
+ * for whatever reason.
  */
-EXPORT void obs_source_process_filter_begin(obs_source_t *filter,
+EXPORT bool obs_source_process_filter_begin(obs_source_t *filter,
 		enum gs_color_format format,
 		enum obs_allow_direct_render allow_direct);
 

plugins/obs-filters/chroma-key-filter.c

@@ -194,8 +194,9 @@ static void chroma_key_render(void *data, gs_effect_t *effect)
 	uint32_t height = obs_source_get_base_height(target);
 	struct vec2 pixel_size;
 
-	obs_source_process_filter_begin(filter->context, GS_RGBA,
-			OBS_ALLOW_DIRECT_RENDERING);
+	if (!obs_source_process_filter_begin(filter->context, GS_RGBA,
+				OBS_ALLOW_DIRECT_RENDERING))
+		return;
 
 	vec2_set(&pixel_size, 1.0f / (float)width, 1.0f / (float)height);
 

plugins/obs-filters/color-filter.c

@@ -119,8 +119,9 @@ static void color_filter_render(void *data, gs_effect_t *effect)
 {
 	struct color_filter_data *filter = data;
 
-	obs_source_process_filter_begin(filter->context, GS_RGBA,
-			OBS_ALLOW_DIRECT_RENDERING);
+	if (!obs_source_process_filter_begin(filter->context, GS_RGBA,
+				OBS_ALLOW_DIRECT_RENDERING))
+		return;
 
 	gs_effect_set_vec4(filter->color_param, &filter->color);
 	gs_effect_set_float(filter->contrast_param, filter->contrast);

plugins/obs-filters/color-key-filter.c

@@ -166,8 +166,9 @@ static void color_key_render(void *data, gs_effect_t *effect)
 {
 	struct color_key_filter_data *filter = data;
 
-	obs_source_process_filter_begin(filter->context, GS_RGBA,
-			OBS_ALLOW_DIRECT_RENDERING);
+	if (!obs_source_process_filter_begin(filter->context, GS_RGBA,
+				OBS_ALLOW_DIRECT_RENDERING))
+		return;
 
 	gs_effect_set_vec4(filter->color_param, &filter->color);
 	gs_effect_set_float(filter->contrast_param, filter->contrast);

plugins/obs-filters/crop-filter.c

@@ -193,8 +193,9 @@ static void crop_filter_render(void *data, gs_effect_t *effect)
 {
 	struct crop_filter_data *filter = data;
 
-	obs_source_process_filter_begin(filter->context, GS_RGBA,
-			OBS_NO_DIRECT_RENDERING);
+	if (!obs_source_process_filter_begin(filter->context, GS_RGBA,
+				OBS_NO_DIRECT_RENDERING))
+		return;
 
 	gs_effect_set_vec2(filter->param_mul, &filter->mul_val);
 	gs_effect_set_vec2(filter->param_add, &filter->add_val);

plugins/obs-filters/mask-filter.c

@@ -210,8 +210,9 @@ static void mask_filter_render(void *data, gs_effect_t *effect)
 		vec2_div(&add_val, &add_val, &mask_size);
 	}
 
-	obs_source_process_filter_begin(filter->context, GS_RGBA,
-			OBS_ALLOW_DIRECT_RENDERING);
+	if (!obs_source_process_filter_begin(filter->context, GS_RGBA,
+				OBS_ALLOW_DIRECT_RENDERING))
+		return;
 
 	param = gs_effect_get_param_by_name(filter->effect, "target");
 	gs_effect_set_texture(param, filter->target);

plugins/obs-filters/scroll-filter.c

@@ -178,8 +178,9 @@ static void scroll_filter_render(void *data, gs_effect_t *effect)
 			(float)cx / (float)base_cx,
 			(float)cy / (float)base_cy);
 
-	obs_source_process_filter_begin(filter->context, GS_RGBA,
-			OBS_NO_DIRECT_RENDERING);
+	if (!obs_source_process_filter_begin(filter->context, GS_RGBA,
+				OBS_NO_DIRECT_RENDERING))
+		return;
 
 	gs_effect_set_vec2(filter->param_add, &filter->offset);
 	gs_effect_set_vec2(filter->param_mul, &mul_val);

plugins/obs-filters/sharpness-filter.c

@@ -77,11 +77,10 @@ static void *sharpness_create(obs_data_t *settings, obs_source_t *context)
 static void sharpness_render(void *data, gs_effect_t *effect)
 {
 	struct sharpness_data *filter = data;
-	if (!filter) return;
-	if (!obs_filter_get_target(filter->context)) return;
 
-	obs_source_process_filter_begin(filter->context, GS_RGBA,
-		OBS_ALLOW_DIRECT_RENDERING);
+	if (!obs_source_process_filter_begin(filter->context, GS_RGBA,
+				OBS_ALLOW_DIRECT_RENDERING))
+		return;
 
 	filter->texwidth =(float)obs_source_get_width(
 			obs_filter_get_target(filter->context));

test/test-input/test-filter.c

@@ -51,8 +51,10 @@ static void filter_render(void *data, gs_effect_t *effect)
 {
 	struct test_filter *tf = data;
 
-	obs_source_process_filter_begin(tf->source, GS_RGBA,
-			OBS_ALLOW_DIRECT_RENDERING);
+	if (!obs_source_process_filter_begin(tf->source, GS_RGBA,
+				OBS_ALLOW_DIRECT_RENDERING))
+		return;
+
 	obs_source_process_filter_end(tf->source, tf->whatever, 0, 0);
 
 	UNUSED_PARAMETER(effect);