From 8c74db9ffc84aa7e961d61f2f85326fcfdd4aba3 Mon Sep 17 00:00:00 2001 From: jp9000 Date: Fri, 4 Apr 2014 23:21:19 -0700 Subject: [PATCH] Add packet interleaving and improve encoder API - Add interleaving of video/audio packets for outputs that are encoded and expect both video and audio data, sorting the packets and sending them to the output when both video and audio is received. - Combine create and initialize callbacks for the encoder API callback interface. --- libobs/obs-encoder.c | 28 ++++---- libobs/obs-encoder.h | 13 +--- libobs/obs-internal.h | 14 +++- libobs/obs-module.c | 3 +- libobs/obs-output.c | 92 ++++++++++++++++++++++++- plugins/obs-ffmpeg/obs-ffmpeg-formats.h | 37 ++++++++++ plugins/obs-ffmpeg/obs-ffmpeg-output.c | 55 ++++----------- plugins/obs-x264/obs-x264.c | 22 +++--- 8 files changed, 175 insertions(+), 89 deletions(-) create mode 100644 plugins/obs-ffmpeg/obs-ffmpeg-formats.h diff --git a/libobs/obs-encoder.c b/libobs/obs-encoder.c index 89ab24e97..0d948b043 100644 --- a/libobs/obs-encoder.c +++ b/libobs/obs-encoder.c @@ -51,14 +51,6 @@ static bool init_encoder(struct obs_encoder *encoder, const char *name, if (encoder->info.defaults) encoder->info.defaults(encoder->settings); - encoder->data = encoder->info.create(encoder->settings, encoder); - - if (!encoder->data) { - pthread_mutex_destroy(&encoder->callbacks_mutex); - obs_data_release(encoder->settings); - return false; - } - pthread_mutex_lock(&obs->data.encoders_mutex); da_push_back(obs->data.encoders, &encoder); pthread_mutex_unlock(&obs->data.encoders_mutex); @@ -188,7 +180,8 @@ static void obs_encoder_actually_destroy(obs_encoder_t encoder) da_free(encoder->outputs); pthread_mutex_unlock(&encoder->outputs_mutex); - encoder->info.destroy(encoder->data); + if (encoder->data) + encoder->info.destroy(encoder->data); obs_data_release(encoder->settings); pthread_mutex_destroy(&encoder->callbacks_mutex); pthread_mutex_destroy(&encoder->outputs_mutex); @@ -265,15 +258,16 @@ void obs_encoder_update(obs_encoder_t encoder, obs_data_t settings) if (!encoder) return; obs_data_apply(encoder->settings, settings); - if (encoder->info.update) + if (encoder->info.update && encoder->data) encoder->info.update(encoder->data, encoder->settings); } bool obs_encoder_get_extra_data(obs_encoder_t encoder, uint8_t **extra_data, size_t *size) { - if (encoder && encoder->info.extra_data) - return encoder->info.extra_data(encoder, extra_data, size); + if (encoder && encoder->info.extra_data && encoder->data) + return encoder->info.extra_data(encoder->data, extra_data, + size); return false; } @@ -293,9 +287,11 @@ bool obs_encoder_initialize(obs_encoder_t encoder) if (encoder->active) return true; - encoder->initialized = encoder->info.initialize(encoder, - encoder->settings); - return encoder->initialized; + if (encoder->data) + encoder->info.destroy(encoder->data); + + encoder->data = encoder->info.create(encoder->settings, encoder); + return encoder->data != NULL; } static inline size_t get_callback_idx( @@ -321,7 +317,7 @@ void obs_encoder_start(obs_encoder_t encoder, bool success = true; bool first = false; - if (!encoder || !new_packet || !encoder->initialized) return; + if (!encoder || !new_packet || !encoder->data) return; pthread_mutex_lock(&encoder->callbacks_mutex); diff --git a/libobs/obs-encoder.h b/libobs/obs-encoder.h index f34957190..459fe05e3 100644 --- a/libobs/obs-encoder.h +++ b/libobs/obs-encoder.h @@ -109,7 +109,8 @@ struct obs_encoder_info { * * @param settings Settings for the encoder * @param encoder OBS encoder context - * @return Data associated with this encoder context + * @return Data associated with this encoder context, or + * NULL if initialization failed. */ void *(*create)(obs_data_t settings, obs_encoder_t encoder); @@ -120,16 +121,6 @@ struct obs_encoder_info { */ void (*destroy)(void *data); - /** - * Initializes the encoder with the specified settings - * - * @param data Data associated with this encoder context - * @param settings Settings for the encoder - * @return true if the encoder settings are valid and the - * encoder is ready to be used, false otherwise - */ - bool (*initialize)(void *data, obs_data_t settings); - /** * Encodes frame(s), and outputs encoded packets as they become * available. diff --git a/libobs/obs-internal.h b/libobs/obs-internal.h index d501021bd..cb14b764c 100644 --- a/libobs/obs-internal.h +++ b/libobs/obs-internal.h @@ -259,6 +259,12 @@ extern void obs_source_video_tick(obs_source_t source, float seconds); /* ------------------------------------------------------------------------- */ /* outputs */ +struct il_packet { + int64_t input_ts_us; + int64_t output_ts_us; + struct encoder_packet packet; +}; + struct obs_output { char *name; void *data; @@ -268,8 +274,13 @@ struct obs_output { signal_handler_t signals; proc_handler_t procs; + bool received_video; + bool received_audio; + int64_t first_video_ts; + int64_t video_offset; + int64_t audio_offset; pthread_mutex_t interleaved_mutex; - DARRAY(struct encoder_packet) interleaved_packets; + DARRAY(struct il_packet) interleaved_packets; bool active; video_t video; @@ -304,7 +315,6 @@ struct obs_encoder { struct obs_encoder_info info; obs_data_t settings; - bool initialized; bool active; uint32_t timebase_num; diff --git a/libobs/obs-module.c b/libobs/obs-module.c index 0e6d883b7..53c87e292 100644 --- a/libobs/obs-module.c +++ b/libobs/obs-module.c @@ -1,5 +1,5 @@ /****************************************************************************** - Copyright (C) 2013 by Hugh Bailey + Copyright (C) 2013-2014 by Hugh Bailey This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -222,7 +222,6 @@ void obs_register_encoder(const struct obs_encoder_info *info) CHECK_REQUIRED_VAL(info, getname, obs_register_encoder); CHECK_REQUIRED_VAL(info, create, obs_register_encoder); CHECK_REQUIRED_VAL(info, destroy, obs_register_encoder); - CHECK_REQUIRED_VAL(info, initialize, obs_register_encoder); CHECK_REQUIRED_VAL(info, encode, obs_register_encoder); REGISTER_OBS_DEF(cur_encoder_info_size, obs_encoder_info, diff --git a/libobs/obs-output.c b/libobs/obs-output.c index 93e11c744..95fe4cb7e 100644 --- a/libobs/obs-output.c +++ b/libobs/obs-output.c @@ -93,10 +93,15 @@ fail: return NULL; } +static inline void free_il_packet(struct il_packet *data) +{ + obs_free_encoder_packet(&data->packet); +} + static inline void free_packets(struct obs_output *output) { for (size_t i = 0; i < output->interleaved_packets.num; i++) - obs_free_encoder_packet(output->interleaved_packets.array+i); + free_il_packet(output->interleaved_packets.array+i); da_free(output->interleaved_packets); } @@ -342,9 +347,89 @@ static inline struct audio_convert_info *get_audio_conversion( return output->audio_conversion_set ? &output->audio_conversion : NULL; } +#define MICROSECOND_DEN 1000000 + +static inline int64_t convert_packet_dts(struct encoder_packet *packet) +{ + return packet->dts * MICROSECOND_DEN / packet->timebase_den; +} + +static bool prepare_interleaved_packet(struct obs_output *output, + struct il_packet *out, struct encoder_packet *packet) +{ + int64_t offset; + + out->input_ts_us = convert_packet_dts(packet); + + /* audio and video need to start at timestamp 0, and the encoders + * may not currently be at 0 when we get data. so, we store the + * current dts as offset and subtract that value from the dts/pts + * of the output packet. */ + if (packet->type == OBS_ENCODER_VIDEO) { + if (!output->received_video) { + output->first_video_ts = out->input_ts_us; + output->video_offset = packet->dts; + output->received_video = true; + } + + offset = output->video_offset; + } else{ + /* don't accept audio that's before the first video timestamp */ + if (!output->received_video || + out->input_ts_us < output->first_video_ts) + return false; + + if (!output->received_audio) { + output->audio_offset = packet->dts; + output->received_audio = true; + } + + offset = output->audio_offset; + } + + obs_duplicate_encoder_packet(&out->packet, packet); + out->packet.dts -= offset; + out->packet.pts -= offset; + out->output_ts_us = convert_packet_dts(&out->packet); + return true; +} + +static inline void send_interleaved(struct obs_output *output) +{ + struct il_packet out = output->interleaved_packets.array[0]; + da_erase(output->interleaved_packets, 0); + + output->info.encoded_packet(output->data, &out.packet); + free_il_packet(&out); +} + static void interleave_packets(void *data, struct encoder_packet *packet) { - + struct obs_output *output = data; + struct il_packet out; + size_t idx; + + pthread_mutex_lock(&output->interleaved_mutex); + + if (!prepare_interleaved_packet(output, &out, packet)) { + + for (idx = 0; idx < output->interleaved_packets.num; idx++) { + struct il_packet *cur_packet; + cur_packet = output->interleaved_packets.array + idx; + + if (out.output_ts_us < cur_packet->output_ts_us) + break; + } + + da_insert(output->interleaved_packets, idx, &out); + + /* when both video and audio have been received, we're ready + * to start sending out packets (one at a time) */ + if (output->received_audio && output->received_video) + send_interleaved(output); + } + + pthread_mutex_unlock(&output->interleaved_mutex); } static void hook_data_capture(struct obs_output *output, bool encoded, @@ -354,6 +439,9 @@ static void hook_data_capture(struct obs_output *output, bool encoded, void *param; if (encoded) { + output->received_video = false; + output->received_video = false; + encoded_callback = (has_video && has_audio) ? interleave_packets : output->info.encoded_packet; param = (has_video && has_audio) ? output : output->data; diff --git a/plugins/obs-ffmpeg/obs-ffmpeg-formats.h b/plugins/obs-ffmpeg/obs-ffmpeg-formats.h new file mode 100644 index 000000000..65a9a9059 --- /dev/null +++ b/plugins/obs-ffmpeg/obs-ffmpeg-formats.h @@ -0,0 +1,37 @@ +#pragma once + +static inline enum AVPixelFormat obs_to_ffmpeg_video_format( + enum video_format format) +{ + switch (format) { + case VIDEO_FORMAT_NONE: return AV_PIX_FMT_NONE; + case VIDEO_FORMAT_I420: return AV_PIX_FMT_YUV420P; + case VIDEO_FORMAT_NV12: return AV_PIX_FMT_NV12; + case VIDEO_FORMAT_YVYU: return AV_PIX_FMT_NONE; + case VIDEO_FORMAT_YUY2: return AV_PIX_FMT_YUYV422; + case VIDEO_FORMAT_UYVY: return AV_PIX_FMT_UYVY422; + case VIDEO_FORMAT_RGBA: return AV_PIX_FMT_RGBA; + case VIDEO_FORMAT_BGRA: return AV_PIX_FMT_BGRA; + case VIDEO_FORMAT_BGRX: return AV_PIX_FMT_BGRA; + } + + return AV_PIX_FMT_NONE; +} + +static inline enum audio_format convert_ffmpeg_sample_format( + enum AVSampleFormat format) +{ + switch ((uint32_t)format) { + case AV_SAMPLE_FMT_U8: return AUDIO_FORMAT_U8BIT; + case AV_SAMPLE_FMT_S16: return AUDIO_FORMAT_16BIT; + case AV_SAMPLE_FMT_S32: return AUDIO_FORMAT_32BIT; + case AV_SAMPLE_FMT_FLT: return AUDIO_FORMAT_FLOAT; + case AV_SAMPLE_FMT_U8P: return AUDIO_FORMAT_U8BIT_PLANAR; + case AV_SAMPLE_FMT_S16P: return AUDIO_FORMAT_16BIT_PLANAR; + case AV_SAMPLE_FMT_S32P: return AUDIO_FORMAT_32BIT_PLANAR; + case AV_SAMPLE_FMT_FLTP: return AUDIO_FORMAT_FLOAT_PLANAR; + } + + /* shouldn't get here */ + return AUDIO_FORMAT_16BIT; +} diff --git a/plugins/obs-ffmpeg/obs-ffmpeg-output.c b/plugins/obs-ffmpeg/obs-ffmpeg-output.c index 02bbc2a28..546a243fc 100644 --- a/plugins/obs-ffmpeg/obs-ffmpeg-output.c +++ b/plugins/obs-ffmpeg/obs-ffmpeg-output.c @@ -26,6 +26,8 @@ #include #include +#include "obs-ffmpeg-formats.h" + //#define OBS_FFMPEG_VIDEO_FORMAT VIDEO_FORMAT_I420 #define OBS_FFMPEG_VIDEO_FORMAT VIDEO_FORMAT_NV12 @@ -82,42 +84,6 @@ struct ffmpeg_output { /* ------------------------------------------------------------------------- */ -static inline enum AVPixelFormat obs_to_ffmpeg_video_format( - enum video_format format) -{ - switch (format) { - case VIDEO_FORMAT_NONE: return AV_PIX_FMT_NONE; - case VIDEO_FORMAT_I420: return AV_PIX_FMT_YUV420P; - case VIDEO_FORMAT_NV12: return AV_PIX_FMT_NV12; - case VIDEO_FORMAT_YVYU: return AV_PIX_FMT_NONE; - case VIDEO_FORMAT_YUY2: return AV_PIX_FMT_YUYV422; - case VIDEO_FORMAT_UYVY: return AV_PIX_FMT_UYVY422; - case VIDEO_FORMAT_RGBA: return AV_PIX_FMT_RGBA; - case VIDEO_FORMAT_BGRA: return AV_PIX_FMT_BGRA; - case VIDEO_FORMAT_BGRX: return AV_PIX_FMT_BGRA; - } - - return AV_PIX_FMT_NONE; -} - -static inline enum audio_format convert_ffmpeg_sample_format( - enum AVSampleFormat format) -{ - switch ((uint32_t)format) { - case AV_SAMPLE_FMT_U8: return AUDIO_FORMAT_U8BIT; - case AV_SAMPLE_FMT_S16: return AUDIO_FORMAT_16BIT; - case AV_SAMPLE_FMT_S32: return AUDIO_FORMAT_32BIT; - case AV_SAMPLE_FMT_FLT: return AUDIO_FORMAT_FLOAT; - case AV_SAMPLE_FMT_U8P: return AUDIO_FORMAT_U8BIT_PLANAR; - case AV_SAMPLE_FMT_S16P: return AUDIO_FORMAT_16BIT_PLANAR; - case AV_SAMPLE_FMT_S32P: return AUDIO_FORMAT_32BIT_PLANAR; - case AV_SAMPLE_FMT_FLTP: return AUDIO_FORMAT_FLOAT_PLANAR; - } - - /* shouldn't get here */ - return AUDIO_FORMAT_16BIT; -} - static bool new_stream(struct ffmpeg_data *data, AVStream **stream, AVCodec **codec, enum AVCodecID id) { @@ -180,7 +146,9 @@ static bool open_video_codec(struct ffmpeg_data *data) static bool init_swscale(struct ffmpeg_data *data, AVCodecContext *context) { - enum AVPixelFormat format = obs_to_ffmpeg_video_format(OBS_FFMPEG_VIDEO_FORMAT); + enum AVPixelFormat format; + format = obs_to_ffmpeg_video_format(OBS_FFMPEG_VIDEO_FORMAT); + data->swscale = sws_getContext( context->width, context->height, format, context->width, context->height, context->pix_fmt, @@ -198,6 +166,9 @@ static bool create_video_stream(struct ffmpeg_data *data) { AVCodecContext *context; struct obs_video_info ovi; + enum AVPixelFormat vformat; + + vformat = obs_to_ffmpeg_video_format(OBS_FFMPEG_VIDEO_FORMAT); if (!obs_get_video_info(&ovi)) { blog(LOG_WARNING, "No active video"); @@ -218,7 +189,7 @@ static bool create_video_stream(struct ffmpeg_data *data) context->time_base.num = ovi.fps_den; context->time_base.den = ovi.fps_num; context->gop_size = 120; - context->pix_fmt = obs_to_ffmpeg_video_format(OBS_FFMPEG_VIDEO_FORMAT); + context->pix_fmt = vformat; if (data->output->oformat->flags & AVFMT_GLOBALHEADER) context->flags |= CODEC_FLAG_GLOBAL_HEADER; @@ -226,8 +197,7 @@ static bool create_video_stream(struct ffmpeg_data *data) if (!open_video_codec(data)) return false; - enum AVPixelFormat format = obs_to_ffmpeg_video_format(OBS_FFMPEG_VIDEO_FORMAT); - if (context->pix_fmt != format) + if (context->pix_fmt != vformat) if (!init_swscale(data, context)) return false; @@ -523,13 +493,14 @@ static void receive_video(void *param, struct video_data *frame) AVCodecContext *context = data->video->codec; AVPacket packet = {0}; int ret = 0, got_packet; + enum AVPixelFormat format; av_init_packet(&packet); if (!data->start_timestamp) data->start_timestamp = frame->timestamp; - enum AVPixelFormat format = obs_to_ffmpeg_video_format(OBS_FFMPEG_VIDEO_FORMAT); + format = obs_to_ffmpeg_video_format(OBS_FFMPEG_VIDEO_FORMAT); if (context->pix_fmt != format) sws_scale(data->swscale, (const uint8_t *const *)frame->data, (const int*)frame->linesize, @@ -585,7 +556,7 @@ static void receive_video(void *param, struct video_data *frame) data->total_frames++; } -static inline void encode_audio(struct ffmpeg_output *output, +static void encode_audio(struct ffmpeg_output *output, struct AVCodecContext *context, size_t block_size) { struct ffmpeg_data *data = &output->ff_data; diff --git a/plugins/obs-x264/obs-x264.c b/plugins/obs-x264/obs-x264.c index e4fb48e6b..a425bef45 100644 --- a/plugins/obs-x264/obs-x264.c +++ b/plugins/obs-x264/obs-x264.c @@ -331,11 +331,10 @@ static void load_headers(struct obs_x264 *obsx264) obsx264->sei_size = sei.num; } -static bool obs_x264_initialize(void *data, obs_data_t settings) +static void *obs_x264_create(obs_data_t settings, obs_encoder_t encoder) { - struct obs_x264 *obsx264 = data; - - clear_data(data); + struct obs_x264 *obsx264 = bzalloc(sizeof(struct obs_x264)); + obsx264->encoder = encoder; if (update_settings(obsx264, settings)) { obsx264->context = x264_encoder_open(&obsx264->params); @@ -348,16 +347,12 @@ static bool obs_x264_initialize(void *data, obs_data_t settings) blog(LOG_WARNING, "bad settings specified for x264"); } - return obsx264->context != NULL; -} - -static void *obs_x264_create(obs_data_t settings, obs_encoder_t encoder) -{ - struct obs_x264 *data = bzalloc(sizeof(struct obs_x264)); - data->encoder = encoder; + if (!obsx264->context) { + bfree(obsx264); + return NULL; + } - UNUSED_PARAMETER(settings); - return data; + return obsx264; } static inline int drop_priority(int priority) @@ -492,7 +487,6 @@ struct obs_encoder_info obs_x264_encoder = { .getname = obs_x264_getname, .create = obs_x264_create, .destroy = obs_x264_destroy, - .initialize = obs_x264_initialize, .encode = obs_x264_encode, .properties = obs_x264_props, .defaults = obs_x264_defaults, -- GitLab