libobs: Fix audio buffer clear in custom source mixing

The memset in custom_audio_render() did not clear all audio buffers when
the number of output channels was less then 8.  This caused wrong audio
output on mixes that did not get cleared.

Closes jp9000/obs-studio#1123

libobs/media-io/audio-io.h

@@ -29,6 +29,10 @@ extern "C" {
 #define MAX_AUDIO_CHANNELS  8
 #define AUDIO_OUTPUT_FRAMES 1024
 
+#define TOTAL_AUDIO_SIZE \
+	(MAX_AUDIO_MIXES * MAX_AUDIO_CHANNELS * \
+	 AUDIO_OUTPUT_FRAMES * sizeof(float))
+
 /*
  * Base audio output component.  Use this to create an audio output track
  * for the media.

libobs/obs-source-transition.c

@@ -878,10 +878,6 @@ static inline uint64_t calc_min_ts(obs_source_t *sources[2])
 	return min_ts;
 }
 
-#define TOTAL_AUDIO_SIZE \
-	(MAX_AUDIO_MIXES * MAX_AUDIO_CHANNELS * \
-	 AUDIO_OUTPUT_FRAMES * sizeof(float))
-
 static inline bool stop_audio(obs_source_t *transition)
 {
 	transition->transitioning_audio = false;

libobs/obs-source.c

@@ -3887,13 +3887,17 @@ static void custom_audio_render(obs_source_t *source, uint32_t mixers,
 	uint64_t ts;
 
 	for (size_t mix = 0; mix < MAX_AUDIO_MIXES; mix++) {
-		for (size_t ch = 0; ch < channels; ch++)
+		for (size_t ch = 0; ch < channels; ch++) {
 			audio_data.output[mix].data[ch] =
 				source->audio_output_buf[mix][ch];
-	}
+		}
 
-	memset(audio_data.output[0].data[0], 0, AUDIO_OUTPUT_FRAMES *
-			MAX_AUDIO_MIXES * channels * sizeof(float));
+		if ((source->audio_mixers & mixers & (1 << mix)) != 0) {
+			memset(source->audio_output_buf[mix][0], 0,
+					sizeof(float) * AUDIO_OUTPUT_FRAMES *
+					channels);
+		}
+	}
 
 	success = source->info.audio_render(source->context.data, &ts,
 			&audio_data, mixers, channels, sample_rate);