From 7dada3bef4786153aa27f0b4ae05f5ffcca01e6a Mon Sep 17 00:00:00 2001 From: jp9000 Date: Sat, 26 Oct 2013 14:32:06 -0700 Subject: [PATCH] fill out the rest of the source video frame functions, added nv12 decompression function, and cleaned up the design of the source video frame stuff --- libobs/media-io/format-conversion.c | 44 ++++++- libobs/media-io/format-conversion.h | 4 + libobs/media-io/video-io.h | 7 +- libobs/obs-source.c | 190 +++++++++++++++++++++++++--- libobs/obs-source.h | 11 +- libobs/obs.h | 17 +-- 6 files changed, 226 insertions(+), 47 deletions(-) diff --git a/libobs/media-io/format-conversion.c b/libobs/media-io/format-conversion.c index 7a274d8b..d9237ba6 100644 --- a/libobs/media-io/format-conversion.c +++ b/libobs/media-io/format-conversion.c @@ -214,6 +214,42 @@ void decompress_420(const void *input_v, uint32_t width, uint32_t height, } } +void decompress_nv12(const void *input_v, uint32_t width, uint32_t height, + uint32_t row_bytes, uint32_t start_y, uint32_t end_y, + void *output_v) +{ + uint8_t *output = output_v; + const uint8_t *input = input_v; + const uint8_t *input2 = input + width * height; + + uint32_t start_y_d2 = start_y/2; + uint32_t width_d2 = width/2; + uint32_t height_d2 = end_y/2; + uint32_t y; + + for (y = start_y_d2; y < height_d2; y++) { + const uint16_t *chroma = (uint16_t*)(input2 + y * width); + register const uint8_t *lum0, *lum1; + register uint32_t *output0, *output1; + uint32_t x; + + lum0 = input + y * 2*width; + lum1 = lum0 + width; + output0 = (uint32_t*)(output + y * 2*row_bytes); + output1 = (uint32_t*)((uint8_t*)output0 + row_bytes); + + for (x = 0; x < width_d2; x++) { + uint32_t out = *(chroma++) << 8; + + *(output0++) = *(lum0++) | out; + *(output0++) = *(lum0++) | out; + + *(output1++) = *(lum1++) | out; + *(output1++) = *(lum1++) | out; + } + } +} + void decompress_422(const void *input_v, uint32_t width, uint32_t height, uint32_t row_bytes, uint32_t start_y, uint32_t end_y, void *output_v, bool leading_lum) @@ -221,8 +257,8 @@ void decompress_422(const void *input_v, uint32_t width, uint32_t height, const uint8_t *input = input_v; uint8_t *output = output_v; - uint32_t width_d2 = width>>1; - uint32_t line_size = width*2; + uint32_t width_d2 = width >> 1; + uint32_t line_size = width * 2; uint32_t y; register const uint32_t *input32; @@ -230,7 +266,7 @@ void decompress_422(const void *input_v, uint32_t width, uint32_t height, register uint32_t *output32; if (leading_lum) { - for (y = 0; y < height; y++) { + for (y = start_y; y < end_y; y++) { input32 = (uint32_t*)(input + y*line_size); input32_end = input32 + width_d2; output32 = (uint32_t*)(output + y*row_bytes); @@ -248,7 +284,7 @@ void decompress_422(const void *input_v, uint32_t width, uint32_t height, } } } else { - for (y = 0; y < height; y++) { + for (y = start_y; y < end_y; y++) { input32 = (uint32_t*)(input + y*line_size); input32_end = input32 + width_d2; output32 = (uint32_t*)(output + y*row_bytes); diff --git a/libobs/media-io/format-conversion.h b/libobs/media-io/format-conversion.h index 3fb6fa38..73dc71d9 100644 --- a/libobs/media-io/format-conversion.h +++ b/libobs/media-io/format-conversion.h @@ -31,6 +31,10 @@ EXPORT void compress_uyvx_to_nv12(const void *input, uint32_t width, uint32_t height, uint32_t row_bytes, uint32_t start_y, uint32_t end_y, void **output); +EXPORT void decompress_nv12(const void *input, + uint32_t width, uint32_t height, uint32_t row_bytes, + uint32_t start_y, uint32_t end_y, void *output); + EXPORT void decompress_420(const void *input, uint32_t width, uint32_t height, uint32_t row_bytes, uint32_t start_y, uint32_t end_y, void *output); diff --git a/libobs/media-io/video-io.h b/libobs/media-io/video-io.h index 70c5bcac..fdb2d074 100644 --- a/libobs/media-io/video-io.h +++ b/libobs/media-io/video-io.h @@ -36,8 +36,8 @@ enum video_type { VIDEO_FORMAT_UNKNOWN, /* planar 420 format */ - VIDEO_FORMAT_I420, /* planar 4:2:0 */ - VIDEO_FORMAT_NV12, /* two-plane lum and packed chroma */ + VIDEO_FORMAT_I420, /* three-plane */ + VIDEO_FORMAT_NV12, /* two-plane, lum and packed chroma */ /* packed 422 formats */ VIDEO_FORMAT_YVYU, @@ -45,7 +45,8 @@ enum video_type { VIDEO_FORMAT_UYVY, /* packed uncompressed formats */ - VIDEO_FORMAT_UYVX, /* packed UYV */ + VIDEO_FORMAT_YUVX, + VIDEO_FORMAT_UYVX, VIDEO_FORMAT_RGBA, VIDEO_FORMAT_BGRA, VIDEO_FORMAT_BGRX, diff --git a/libobs/obs-source.c b/libobs/obs-source.c index 70072d9b..73ba4125 100644 --- a/libobs/obs-source.c +++ b/libobs/obs-source.c @@ -15,6 +15,7 @@ along with this program. If not, see . ******************************************************************************/ +#include "media-io\format-conversion.h" #include "util/platform.h" #include "obs.h" @@ -297,20 +298,111 @@ static bool set_texture_size(obs_source_t source, struct source_frame *frame) return source->output_texture != NULL; } +enum convert_type { + CONVERT_NONE, + CONVERT_NV12, + CONVERT_420, + CONVERT_422_U, + CONVERT_422_Y, +}; + +static inline enum convert_type get_convert_type(enum video_type type) +{ + switch (type) { + case VIDEO_FORMAT_I420: + return CONVERT_420; + case VIDEO_FORMAT_NV12: + return CONVERT_NV12; + + case VIDEO_FORMAT_YVYU: + case VIDEO_FORMAT_YUY2: + return CONVERT_422_Y; + case VIDEO_FORMAT_UYVY: + return CONVERT_422_U; + + case VIDEO_FORMAT_UNKNOWN: + case VIDEO_FORMAT_YUVX: + case VIDEO_FORMAT_UYVX: + case VIDEO_FORMAT_RGBA: + case VIDEO_FORMAT_BGRA: + case VIDEO_FORMAT_BGRX: + return CONVERT_NONE; + } + + return CONVERT_NONE; +} + +static inline bool is_yuv(enum video_type type) +{ + switch (type) { + case VIDEO_FORMAT_I420: + case VIDEO_FORMAT_NV12: + case VIDEO_FORMAT_YVYU: + case VIDEO_FORMAT_YUY2: + case VIDEO_FORMAT_UYVY: + case VIDEO_FORMAT_YUVX: + case VIDEO_FORMAT_UYVX: + return true; + case VIDEO_FORMAT_UNKNOWN: + case VIDEO_FORMAT_RGBA: + case VIDEO_FORMAT_BGRA: + case VIDEO_FORMAT_BGRX: + return false; + } + + return false; +} + +static bool upload_frame(texture_t tex, const struct source_frame *frame) +{ + void *ptr; + uint32_t row_bytes; + enum convert_type type = get_convert_type(frame->type); + + if (type == CONVERT_NONE) { + texture_setimage(tex, frame->data, frame->row_bytes, false); + return true; + } + + if (!texture_map(tex, &ptr, &row_bytes)) + return false; + + if (type == CONVERT_420) + decompress_420(frame->data, frame->width, frame->height, + frame->row_bytes, 0, frame->height, ptr); + + else if (type == CONVERT_NV12) + decompress_nv12(frame->data, frame->width, frame->height, + frame->row_bytes, 0, frame->height, ptr); + + else if (type == CONVERT_422_Y) + decompress_422(frame->data, frame->width, frame->height, + frame->row_bytes, 0, frame->height, ptr, true); + + else if (type == CONVERT_422_U) + decompress_422(frame->data, frame->width, frame->height, + frame->row_bytes, 0, frame->height, ptr, false); + + texture_unmap(tex); + return true; +} + static void obs_source_draw_texture(texture_t tex, struct source_frame *frame) { effect_t effect = obs->default_effect; - const char *type = frame->yuv ? "DrawYUVToRGB" : "DrawRGB"; + bool yuv = is_yuv(frame->type); + const char *type = yuv ? "DrawYUVToRGB" : "DrawRGB"; technique_t tech; - eparam_t param; + eparam_t param; - texture_setimage(tex, frame->data, frame->row_bytes, frame->flip); + if (!upload_frame(tex, frame)) + return; tech = effect_gettechnique(effect, type); technique_begin(tech); technique_beginpass(tech, 0); - if (frame->yuv) { + if (yuv) { param = effect_getparambyname(effect, "yuv_matrix"); effect_setval(effect, param, frame->yuv_matrix, sizeof(float) * 16); @@ -335,8 +427,10 @@ static void obs_source_render_async_video(obs_source_t source) if (!source->timing_set && source->audio_buffer.num) obs_source_flush_audio_buffer(source); - if (set_texture_size(source, frame)) + if (set_texture_size(source, frame)) { + source->flip = frame->flip; obs_source_draw_texture(source->output_texture, frame); + } obs_source_releaseframe(source, frame); } @@ -496,8 +590,8 @@ void obs_source_save_settings(obs_source_t source, const char *settings) dstr_copy(&source->settings, settings); } -static inline struct filter_frame *filter_async_video(obs_source_t source, - struct filter_frame *in) +static inline struct source_frame *filter_async_video(obs_source_t source, + struct source_frame *in) { size_t i; for (i = source->filters.num; i > 0; i--) { @@ -512,27 +606,31 @@ static inline struct filter_frame *filter_async_video(obs_source_t source, return in; } -static struct filter_frame *process_video(obs_source_t source, - const struct source_video *frame) +static inline struct source_frame *cache_video(obs_source_t source, + const struct source_frame *frame) { - /* TODO: convert to UYV444 or RGB */ - return NULL; + /* TODO: use an actual cache */ + struct source_frame *new_frame = bmalloc(sizeof(struct source_frame)); + memcpy(new_frame, frame, sizeof(struct source_frame)); + new_frame->data = bmalloc(frame->row_bytes * frame->height); + + return new_frame; } void obs_source_output_video(obs_source_t source, - const struct source_video *frame) + const struct source_frame *frame) { - struct filter_frame *output; - - output = process_video(source, frame); + struct source_frame *output = cache_video(source, frame); pthread_mutex_lock(&source->filter_mutex); output = filter_async_video(source, output); pthread_mutex_unlock(&source->filter_mutex); - pthread_mutex_lock(&source->video_mutex); - da_push_back(source->video_frames, &output); - pthread_mutex_unlock(&source->video_mutex); + if (output) { + pthread_mutex_lock(&source->video_mutex); + da_push_back(source->video_frames, &output); + pthread_mutex_unlock(&source->video_mutex); + } } static inline const struct audio_data *filter_async_audio(obs_source_t source, @@ -595,13 +693,63 @@ void obs_source_output_audio(obs_source_t source, pthread_mutex_unlock(&source->filter_mutex); } +/* + * Ensures that cached frames are displayed on time. If multiple frames + * were cached between renders, then releases the unnecessary frames and uses + * the frame with the closest timing. + */ struct source_frame *obs_source_getframe(obs_source_t source) { - /* TODO */ - return NULL; + uint64_t last_frame_time = source->last_frame_timestamp; + struct source_frame *frame = NULL; + struct source_frame *next_frame; + uint64_t sys_time, frame_time; + + pthread_mutex_lock(&source->video_mutex); + + if (!source->video_frames.num) + goto unlock; + + next_frame = source->video_frames.array[0]; + sys_time = os_gettime_ns(); + frame_time = next_frame->timestamp; + + if (!source->last_frame_timestamp) { + frame = next_frame; + da_erase(source->video_frames, 0); + + source->last_frame_timestamp = frame_time; + } else { + uint64_t sys_offset, frame_offset; + sys_offset = sys_time - source->last_sys_timestamp; + frame_offset = frame_time - last_frame_time; + + source->last_frame_timestamp += sys_offset; + + while (frame_offset <= sys_offset) { + if (frame) + source_frame_destroy(frame); + + frame = next_frame; + da_erase(source->video_frames, 0); + + if (!source->video_frames.num) + break; + + next_frame = source->video_frames.array[0]; + frame_time = next_frame->timestamp; + frame_offset = frame_time - last_frame_time; + } + } + + source->last_sys_timestamp = sys_time; + +unlock: + pthread_mutex_unlock(&source->video_mutex); + return frame; } void obs_source_releaseframe(obs_source_t source, struct source_frame *frame) { - /* TODO */ + source_frame_destroy(frame); } diff --git a/libobs/obs-source.h b/libobs/obs-source.h index 09c1213e..046abf61 100644 --- a/libobs/obs-source.h +++ b/libobs/obs-source.h @@ -136,8 +136,8 @@ * Return value: true if sources remaining, otherwise false. * * --------------------------------------------------------- - * struct filter_frame *[name]_filter_video(void *data, - * struct filter_frame *frame); + * struct source_frame *[name]_filter_video(void *data, + * const struct source_frame *frame); * Filters audio data. Used with audio filters. * * frame: Video frame data. @@ -185,8 +185,8 @@ struct source_info { bool (*enum_children)(void *data, size_t idx, obs_source_t *child); - struct filter_frame *(*filter_video)(void *data, - struct filter_frame *frame); + struct source_frame *(*filter_video)(void *data, + const struct source_frame *frame); const struct audio_data *(*filter_audio)(void *data, const struct audio_data *audio); }; @@ -211,7 +211,10 @@ struct obs_source { /* async video and audio */ bool timing_set; uint64_t timing_adjust; + uint64_t last_frame_timestamp; + uint64_t last_sys_timestamp; texture_t output_texture; + bool flip; audio_line_t audio_line; DARRAY(struct audiobuf) audio_buffer; diff --git a/libobs/obs.h b/libobs/obs.h index e0e69fab..9ea86518 100644 --- a/libobs/obs.h +++ b/libobs/obs.h @@ -61,19 +61,6 @@ struct source_audio { uint64_t timestamp; }; -struct source_video { - const void *data; - uint32_t width; - uint32_t height; - uint32_t row_bytes; - uint64_t timestamp; - - enum video_type type; - float yuv_matrix[16]; - bool flip; -}; - -/* differs from source_video in that it's YUV444 or RGB only */ struct source_frame { void *data; uint32_t width; @@ -81,7 +68,7 @@ struct source_frame { uint32_t row_bytes; uint64_t timestamp; - bool yuv; + enum video_type type; float yuv_matrix[16]; bool flip; }; @@ -291,7 +278,7 @@ EXPORT void obs_source_save_settings(obs_source_t source, const char *settings); /** Outputs asynchronous video data */ EXPORT void obs_source_obs_async_video(obs_source_t source, - const struct video_frame *frame); + const struct source_frame *frame); /** Outputs audio data (always asynchronous) */ EXPORT void obs_source_obs_async_audio(obs_source_t source, -- GitLab