From f0271e54d90b3af62301f531f5f00995b00d7cd6 Mon Sep 17 00:00:00 2001 From: Alexey Lapshin Date: Tue, 2 Jun 2020 18:34:52 +0300 Subject: [PATCH] Autorotation for mp4 streams with metadata - Add VideoCapture camera orientation property for mp4 videos with camera orientation meta. - Add auto rotation for 90, 180, 270 degrees using cv::rotate --- modules/videoio/include/opencv2/videoio.hpp | 2 + modules/videoio/src/cap_ffmpeg.cpp | 11 ++-- modules/videoio/src/cap_ffmpeg_api.hpp | 10 +-- modules/videoio/src/cap_ffmpeg_impl.hpp | 73 +++++++++++++++------ 4 files changed, 67 insertions(+), 29 deletions(-) diff --git a/modules/videoio/include/opencv2/videoio.hpp b/modules/videoio/include/opencv2/videoio.hpp index 38a451b244..8b98ca9a17 100644 --- a/modules/videoio/include/opencv2/videoio.hpp +++ b/modules/videoio/include/opencv2/videoio.hpp @@ -177,6 +177,8 @@ enum VideoCaptureProperties { CAP_PROP_WB_TEMPERATURE=45, //!< white-balance color temperature CAP_PROP_CODEC_PIXEL_FORMAT =46, //!< (read-only) codec's pixel format. 4-character code - see VideoWriter::fourcc . Subset of [AV_PIX_FMT_*](https://github.com/FFmpeg/FFmpeg/blob/master/libavcodec/raw.c) or -1 if unknown CAP_PROP_BITRATE =47, //!< (read-only) Video bitrate in kbits/s + CAP_PROP_ORIENTATION_META=48, //!< (read-only) Frame rotation defined by stream meta (applicable for FFmpeg back-end only) + CAP_PROP_ORIENTATION_AUTO=49, //!< if true - rotates output frames of CvCapture considering video file's metadata (applicable for FFmpeg back-end only) (https://github.com/opencv/opencv/issues/15499) #ifndef CV_DOXYGEN CV__CAP_PROP_LATEST #endif diff --git a/modules/videoio/src/cap_ffmpeg.cpp b/modules/videoio/src/cap_ffmpeg.cpp index 67ee2e636d..a4aed6c741 100644 --- a/modules/videoio/src/cap_ffmpeg.cpp +++ b/modules/videoio/src/cap_ffmpeg.cpp @@ -229,13 +229,12 @@ public: } virtual bool retrieveFrame(int, cv::OutputArray frame) CV_OVERRIDE { - unsigned char* data = 0; - int step=0, width=0, height=0, cn=0; - + cv::Mat mat; if (!ffmpegCapture || - !icvRetrieveFrame_FFMPEG_p(ffmpegCapture, &data, &step, &width, &height, &cn)) + !icvRetrieveFrame_FFMPEG_p(ffmpegCapture, mat)) return false; - cv::Mat(height, width, CV_MAKETYPE(CV_8U, cn), data, step).copyTo(frame); + + mat.copyTo(frame); return true; } virtual bool open( const cv::String& filename ) @@ -262,6 +261,8 @@ public: protected: CvCapture_FFMPEG* ffmpegCapture; +private: + }; } // namespace diff --git a/modules/videoio/src/cap_ffmpeg_api.hpp b/modules/videoio/src/cap_ffmpeg_api.hpp index 004758099c..c132fa1d4f 100644 --- a/modules/videoio/src/cap_ffmpeg_api.hpp +++ b/modules/videoio/src/cap_ffmpeg_api.hpp @@ -28,7 +28,9 @@ enum CV_FFMPEG_CAP_PROP_SAR_NUM=40, CV_FFMPEG_CAP_PROP_SAR_DEN=41, CV_FFMPEG_CAP_PROP_CODEC_PIXEL_FORMAT=46, - CV_FFMPEG_CAP_PROP_BITRATE=47 + CV_FFMPEG_CAP_PROP_BITRATE=47, + CV_FFMPEG_CAP_PROP_ORIENTATION_META=48, + CV_FFMPEG_CAP_PROP_ORIENTATION_AUTO=49 }; typedef struct CvCapture_FFMPEG CvCapture_FFMPEG; @@ -39,8 +41,7 @@ OPENCV_FFMPEG_API int cvSetCaptureProperty_FFMPEG(struct CvCapture_FFMPEG* cap, int prop, double value); OPENCV_FFMPEG_API double cvGetCaptureProperty_FFMPEG(struct CvCapture_FFMPEG* cap, int prop); OPENCV_FFMPEG_API int cvGrabFrame_FFMPEG(struct CvCapture_FFMPEG* cap); -OPENCV_FFMPEG_API int cvRetrieveFrame_FFMPEG(struct CvCapture_FFMPEG* capture, unsigned char** data, - int* step, int* width, int* height, int* cn); +OPENCV_FFMPEG_API int cvRetrieveFrame_FFMPEG(struct CvCapture_FFMPEG* capture, cv::Mat &mat); OPENCV_FFMPEG_API void cvReleaseCapture_FFMPEG(struct CvCapture_FFMPEG** cap); OPENCV_FFMPEG_API struct CvVideoWriter_FFMPEG* cvCreateVideoWriter_FFMPEG(const char* filename, @@ -52,8 +53,7 @@ OPENCV_FFMPEG_API void cvReleaseVideoWriter_FFMPEG(struct CvVideoWriter_FFMPEG** typedef CvCapture_FFMPEG* (*CvCreateFileCapture_Plugin)( const char* filename ); typedef CvCapture_FFMPEG* (*CvCreateCameraCapture_Plugin)( int index ); typedef int (*CvGrabFrame_Plugin)( CvCapture_FFMPEG* capture_handle ); -typedef int (*CvRetrieveFrame_Plugin)( CvCapture_FFMPEG* capture_handle, unsigned char** data, int* step, - int* width, int* height, int* cn ); +typedef int (*CvRetrieveFrame_Plugin)( CvCapture_FFMPEG* capture_handle, cv::Mat &mat); typedef int (*CvSetCaptureProperty_Plugin)( CvCapture_FFMPEG* capture_handle, int prop_id, double value ); typedef double (*CvGetCaptureProperty_Plugin)( CvCapture_FFMPEG* capture_handle, int prop_id ); typedef void (*CvReleaseCapture_Plugin)( CvCapture_FFMPEG** capture_handle ); diff --git a/modules/videoio/src/cap_ffmpeg_impl.hpp b/modules/videoio/src/cap_ffmpeg_impl.hpp index fae85425b0..2b946a738e 100644 --- a/modules/videoio/src/cap_ffmpeg_impl.hpp +++ b/modules/videoio/src/cap_ffmpeg_impl.hpp @@ -481,7 +481,8 @@ struct CvCapture_FFMPEG double getProperty(int) const; bool setProperty(int, double); bool grabFrame(); - bool retrieveFrame(int, unsigned char** data, int* step, int* width, int* height, int* cn); + bool retrieveFrame(int, cv::Mat &mat); + void rotateFrame(cv::Mat &mat) const; void init(); @@ -497,6 +498,7 @@ struct CvCapture_FFMPEG double r2d(AVRational r) const; int64_t dts_to_frame_number(int64_t dts); double dts_to_sec(int64_t dts) const; + void get_rotation_angle(); AVFormatContext * ic; AVCodec * avcodec; @@ -512,6 +514,8 @@ struct CvCapture_FFMPEG int64_t frame_number, first_frame_number; + bool rotation_auto; + int rotation_angle; // valid 0, 90, 180, 270 double eps_zero; /* 'filename' contains the filename of the videosource, @@ -560,6 +564,9 @@ void CvCapture_FFMPEG::init() frame_number = 0; eps_zero = 0.000025; + rotation_auto = true; + rotation_angle = 0; + #if LIBAVFORMAT_BUILD >= CALC_FFMPEG_VERSION(52, 111, 0) dict = NULL; #endif @@ -1032,6 +1039,7 @@ bool CvCapture_FFMPEG::open( const char* _filename ) frame.cn = 3; frame.step = 0; frame.data = NULL; + get_rotation_angle(); break; } } @@ -1279,8 +1287,7 @@ bool CvCapture_FFMPEG::grabFrame() return valid; } - -bool CvCapture_FFMPEG::retrieveFrame(int, unsigned char** data, int* step, int* width, int* height, int* cn) +bool CvCapture_FFMPEG::retrieveFrame(int, cv::Mat &mat) { if (!video_st) return false; @@ -1288,12 +1295,11 @@ bool CvCapture_FFMPEG::retrieveFrame(int, unsigned char** data, int* step, int* if (rawMode) { AVPacket& p = bsfc ? packet_filtered : packet; - *data = p.data; - *step = p.size; - *width = p.size; - *height = 1; - *cn = 1; - return p.data != NULL; + if (p.data == NULL) + return false; + + mat = cv::Mat(1, p.size, CV_MAKETYPE(CV_8U, 1), p.data, p.size); + return true; } if (!picture->data[0]) @@ -1356,15 +1362,29 @@ bool CvCapture_FFMPEG::retrieveFrame(int, unsigned char** data, int* step, int* rgb_picture.linesize ); - *data = frame.data; - *step = frame.step; - *width = frame.width; - *height = frame.height; - *cn = frame.cn; - + mat = cv::Mat(frame.height, frame.width, CV_MAKETYPE(CV_8U, frame.cn), frame.data, frame.step); + rotateFrame(mat); return true; } +void CvCapture_FFMPEG::rotateFrame(cv::Mat &mat) const { + if(!rotation_auto || rotation_angle%360 == 0) { + return; + } + + cv::RotateFlags flag; + if(rotation_angle == 90 || rotation_angle == -270) { // Rotate clockwise 90 degrees + flag = cv::ROTATE_90_CLOCKWISE; + } else if(rotation_angle == 270 || rotation_angle == -90) { // Rotate clockwise 270 degrees + flag = cv::ROTATE_90_COUNTERCLOCKWISE; + } else if(rotation_angle == 180 || rotation_angle == -180) { // Rotate clockwise 180 degrees + flag = cv::ROTATE_180; + } else { // Unsupported rotation + return; + } + + cv::rotate(mat, mat, flag); +} double CvCapture_FFMPEG::getProperty( int property_id ) const { @@ -1389,9 +1409,9 @@ double CvCapture_FFMPEG::getProperty( int property_id ) const case CV_FFMPEG_CAP_PROP_FRAME_COUNT: return (double)get_total_frames(); case CV_FFMPEG_CAP_PROP_FRAME_WIDTH: - return (double)frame.width; + return (double)((rotation_auto && rotation_angle%360) ? frame.height : frame.width); case CV_FFMPEG_CAP_PROP_FRAME_HEIGHT: - return (double)frame.height; + return (double)((rotation_auto && rotation_angle%360) ? frame.width : frame.height); case CV_FFMPEG_CAP_PROP_FPS: return get_fps(); case CV_FFMPEG_CAP_PROP_FOURCC: @@ -1435,6 +1455,10 @@ double CvCapture_FFMPEG::getProperty( int property_id ) const break; case CV_FFMPEG_CAP_PROP_BITRATE: return static_cast(get_bitrate()); + case CV_FFMPEG_CAP_PROP_ORIENTATION_META: + return static_cast(rotation_angle); + case CV_FFMPEG_CAP_PROP_ORIENTATION_AUTO: + return static_cast(rotation_auto); default: break; } @@ -1513,6 +1537,14 @@ double CvCapture_FFMPEG::dts_to_sec(int64_t dts) const r2d(ic->streams[video_stream]->time_base); } +void CvCapture_FFMPEG::get_rotation_angle() +{ + rotation_angle = 0; + AVDictionaryEntry *rotate_tag = av_dict_get(video_st->metadata, "rotate", NULL, 0); + if (rotate_tag != NULL) + rotation_angle = atoi(rotate_tag->value); +} + void CvCapture_FFMPEG::seek(int64_t _frame_number) { _frame_number = std::min(_frame_number, get_total_frames()); @@ -1608,6 +1640,9 @@ bool CvCapture_FFMPEG::setProperty( int property_id, double value ) if (value == -1) return setRaw(); return false; + case CV_FFMPEG_CAP_PROP_ORIENTATION_AUTO: + rotation_auto = static_cast(value); + break; default: return false; } @@ -2635,9 +2670,9 @@ int cvGrabFrame_FFMPEG(CvCapture_FFMPEG* capture) return capture->grabFrame(); } -int cvRetrieveFrame_FFMPEG(CvCapture_FFMPEG* capture, unsigned char** data, int* step, int* width, int* height, int* cn) +int cvRetrieveFrame_FFMPEG(CvCapture_FFMPEG* capture, cv::Mat &mat) { - return capture->retrieveFrame(0, data, step, width, height, cn); + return capture->retrieveFrame(0, mat); } CvVideoWriter_FFMPEG* cvCreateVideoWriter_FFMPEG( const char* filename, int fourcc, double fps, -- GitLab