ios: add yuv420p10le renderer

ijkmedia/ijkplayer/ff_ffplay.c

@@ -3300,6 +3300,11 @@ void ffp_set_overlay_format(FFPlayer *ffp, int chroma_fourcc)
         case SDL_FCC_RV32:
             ffp->overlay_format = chroma_fourcc;
             break;
+#ifdef __APPLE__
+        case SDL_FCC_I444P10LE:
+            ffp->overlay_format = chroma_fourcc;
+            break;
+#endif
         default:
             av_log(ffp, AV_LOG_ERROR, "ffp_set_overlay_format: unknown chroma fourcc: %d\n", chroma_fourcc);
             break;

ijkmedia/ijkplayer/ff_ffplay_options.h

@@ -86,6 +86,9 @@ static const AVOption ffp_context_options[] = {
         OPTION_OFFSET(overlay_format),  OPTION_INT(SDL_FCC_RV32, INT_MIN, INT_MAX),
         .unit = "overlay-format" },
     { "fcc-i420",                       "", 0, OPTION_CONST(SDL_FCC_I420), .unit = "overlay-format" },
+#ifdef __APPLE__
+    { "fcc-i4al",                       "", 0, OPTION_CONST(SDL_FCC_I444P10LE), .unit = "overlay-format" },
+#endif
     { "fcc-yv12",                       "", 0, OPTION_CONST(SDL_FCC_YV12), .unit = "overlay-format" },
     { "fcc-rv16",                       "", 0, OPTION_CONST(SDL_FCC_RV16), .unit = "overlay-format" },
     { "fcc-rv24",                       "", 0, OPTION_CONST(SDL_FCC_RV24), .unit = "overlay-format" },

ijkmedia/ijksdl/ffmpeg/ijksdl_vout_overlay_ffmpeg.c

@@ -182,6 +182,18 @@ static int func_fill_frame(SDL_VoutOverlay *overlay, const AVFrame *frame)
                 dst_format = AV_PIX_FMT_YUV420P;
             }
             break;
+#ifdef __APPLE__
+        case SDL_FCC_I444P10LE:
+            if (frame->format == AV_PIX_FMT_YUV444P10LE) {
+                // ALOGE("direct draw frame");
+                use_linked_frame = 1;
+                dst_format = frame->format;
+            } else {
+                // ALOGE("copy draw frame");
+                dst_format = AV_PIX_FMT_YUV444P10LE;
+            }
+            break;
+#endif
         case SDL_FCC_RV32:
             dst_format = AV_PIX_FMT_0BGR32;
             break;
@@ -318,6 +330,28 @@ SDL_VoutOverlay *SDL_VoutFFmpeg_CreateOverlay(int width, int height, Uint32 form
         opaque->planes = 3;
         break;
     }
+#ifdef __APPLE__
+    case SDL_FCC_I444P10LE: {
+            if (format == SDL_FCC_I444P10LE)
+                ff_format = AV_PIX_FMT_YUV444P10LE;
+            else
+                ff_format = AV_PIX_FMT_YUV420P;
+            // FIXME: need runtime config
+#if defined(__ANDROID__)
+            // 16 bytes align pitch for arm-neon image-convert
+            buf_width = IJKALIGN(width, 16); // 1 bytes per pixel for Y-plane
+#elif defined(__APPLE__)
+            // 2^n align for width
+            buf_width = width;
+            if (width > 0)
+                buf_width = 1 << (sizeof(int) * 8 - __builtin_clz(width));
+#else
+            buf_width = IJKALIGN(width, 16); // unknown platform
+#endif
+            opaque->planes = 3;
+            break;
+        }
+#endif
     case SDL_FCC_RV16: {
         ff_format = AV_PIX_FMT_RGB565;
         buf_width = IJKALIGN(width, 8); // 2 bytes per pixel

ijkmedia/ijksdl/ijksdl_fourcc.h

@@ -49,6 +49,7 @@
 #define SDL_FCC_YV12    SDL_FOURCC('Y', 'V', '1', '2')  /**< bpp=12, Planar mode: Y + V + U  (3 planes) */
 #define SDL_FCC_IYUV    SDL_FOURCC('I', 'Y', 'U', 'V')  /**< bpp=12, Planar mode: Y + U + V  (3 planes) */
 #define SDL_FCC_I420    SDL_FOURCC('I', '4', '2', '0')  /**< bpp=12, Planar mode: Y + U + V  (3 planes) */
+#define SDL_FCC_I444P10LE   SDL_FOURCC('I', '4', 'A', 'L')
 
 #define SDL_FCC_YUV2    SDL_FOURCC('Y', 'U', 'V', '2')  /**< bpp=16, Packed mode: Y0+U0+Y1+V0 (1 plane) */
 #define SDL_FCC_UYVY    SDL_FOURCC('U', 'Y', 'V', 'Y')  /**< bpp=16, Packed mode: U0+Y0+V0+Y1 (1 plane) */

ios/IJKMediaPlayer/IJKMediaPlayer.xcodeproj/project.pbxproj

@@ -70,6 +70,8 @@
 		E654EAEA1B6B295200B0F2D0 /* IJKFFMoviePlayerController.h in Headers */ = {isa = PBXBuildFile; fileRef = E66F8DE517EFD9C300354D80 /* IJKFFMoviePlayerController.h */; settings = {ATTRIBUTES = (Public, ); }; };
 		E654EAEC1B6B295200B0F2D0 /* IJKFFOptions.h in Headers */ = {isa = PBXBuildFile; fileRef = E62139BC180FA89A00553533 /* IJKFFOptions.h */; settings = {ATTRIBUTES = (Public, ); }; };
 		E654EAED1B6B29C100B0F2D0 /* IJKMediaPlayer.h in Headers */ = {isa = PBXBuildFile; fileRef = E66F8DC217EECB1E00354D80 /* IJKMediaPlayer.h */; settings = {ATTRIBUTES = (Public, ); }; };
+		E67F82EB1C27E66900501D6C /* IJKSDLGLRenderI444P10LE.h in Headers */ = {isa = PBXBuildFile; fileRef = E67F82E91C27E66900501D6C /* IJKSDLGLRenderI444P10LE.h */; };
+		E67F82EC1C27E66900501D6C /* IJKSDLGLRenderI444P10LE.m in Sources */ = {isa = PBXBuildFile; fileRef = E67F82EA1C27E66900501D6C /* IJKSDLGLRenderI444P10LE.m */; };
 		E68B7AC51C1E7F20001DE241 /* IJKSDLHudViewController.h in Headers */ = {isa = PBXBuildFile; fileRef = E68B7AC31C1E7F20001DE241 /* IJKSDLHudViewController.h */; };
 		E68B7AC61C1E7F20001DE241 /* IJKSDLHudViewController.m in Sources */ = {isa = PBXBuildFile; fileRef = E68B7AC41C1E7F20001DE241 /* IJKSDLHudViewController.m */; };
 		E68B7ACF1C1E97B0001DE241 /* IJKSDLHudViewCell.h in Headers */ = {isa = PBXBuildFile; fileRef = E68B7ACD1C1E97B0001DE241 /* IJKSDLHudViewCell.h */; };
@@ -158,6 +160,8 @@
 		E67C4E0419D15B3200415CEE /* IJKAVPlayerLayerView.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; name = IJKAVPlayerLayerView.m; path = IJKMediaPlayer/IJKAVPlayerLayerView.m; sourceTree = "<group>"; };
 		E67C4E0619D15EEA00415CEE /* IJKAVMoviePlayerController.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = IJKAVMoviePlayerController.h; path = IJKMediaPlayer/IJKAVMoviePlayerController.h; sourceTree = "<group>"; };
 		E67C4E0719D15EEA00415CEE /* IJKAVMoviePlayerController.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; name = IJKAVMoviePlayerController.m; path = IJKMediaPlayer/IJKAVMoviePlayerController.m; sourceTree = "<group>"; };
+		E67F82E91C27E66900501D6C /* IJKSDLGLRenderI444P10LE.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = IJKSDLGLRenderI444P10LE.h; sourceTree = "<group>"; };
+		E67F82EA1C27E66900501D6C /* IJKSDLGLRenderI444P10LE.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = IJKSDLGLRenderI444P10LE.m; sourceTree = "<group>"; };
 		E67FB4AC1B4A766F00AA94AA /* config.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = config.h; sourceTree = "<group>"; };
 		E68B7AC31C1E7F20001DE241 /* IJKSDLHudViewController.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = IJKSDLHudViewController.h; sourceTree = "<group>"; };
 		E68B7AC41C1E7F20001DE241 /* IJKSDLHudViewController.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = IJKSDLHudViewController.m; sourceTree = "<group>"; };
@@ -604,6 +608,8 @@
 				E6EE92AF1878230C009EAB56 /* IJKSDLGLRender.h */,
 				E6EE92B01878230C009EAB56 /* IJKSDLGLRenderI420.h */,
 				E6EE92B11878230C009EAB56 /* IJKSDLGLRenderI420.m */,
+				E67F82E91C27E66900501D6C /* IJKSDLGLRenderI444P10LE.h */,
+				E67F82EA1C27E66900501D6C /* IJKSDLGLRenderI444P10LE.m */,
 				451103111A5CE36A00337D24 /* IJKSDLGLRenderNV12.h */,
 				451103121A5CE36A00337D24 /* IJKSDLGLRenderNV12.m */,
 				E6EE92B21878230C009EAB56 /* IJKSDLGLRenderRV24.h */,
@@ -612,10 +618,10 @@
 				E6EE92B51878230C009EAB56 /* IJKSDLGLShader.m */,
 				E6EE92B61878230C009EAB56 /* IJKSDLGLView.h */,
 				E6EE92B71878230C009EAB56 /* IJKSDLGLView.m */,
-				E68B7AC31C1E7F20001DE241 /* IJKSDLHudViewController.h */,
-				E68B7AC41C1E7F20001DE241 /* IJKSDLHudViewController.m */,
 				E68B7ACD1C1E97B0001DE241 /* IJKSDLHudViewCell.h */,
 				E68B7ACE1C1E97B0001DE241 /* IJKSDLHudViewCell.m */,
+				E68B7AC31C1E7F20001DE241 /* IJKSDLHudViewController.h */,
+				E68B7AC41C1E7F20001DE241 /* IJKSDLHudViewController.m */,
 			);
 			name = ios;
 			path = ../../ios/IJKMediaPlayer/IJKMediaPlayer/ijkmedia/ijksdl/ios;
@@ -655,6 +661,7 @@
 				E654EAE81B6B295200B0F2D0 /* IJKAVMoviePlayerController.h in Headers */,
 				E654EAEA1B6B295200B0F2D0 /* IJKFFMoviePlayerController.h in Headers */,
 				E654EAEC1B6B295200B0F2D0 /* IJKFFOptions.h in Headers */,
+				E67F82EB1C27E66900501D6C /* IJKSDLGLRenderI444P10LE.h in Headers */,
 				E654EA8F1B6B27E600B0F2D0 /* IJKMediaFramework.h in Headers */,
 				E654EAE61B6B295200B0F2D0 /* IJKMediaModule.h in Headers */,
 				E69BE5571B93FED300AFBA3F /* opt.h in Headers */,
@@ -772,6 +779,7 @@
 				E654EABB1B6B286B00B0F2D0 /* ffpipenode_ffplay_vdec.c in Sources */,
 				E654EACF1B6B288A00B0F2D0 /* IJKSDLGLRenderI420.m in Sources */,
 				E654EAD21B6B288A00B0F2D0 /* IJKSDLGLShader.m in Sources */,
+				E67F82EC1C27E66900501D6C /* IJKSDLGLRenderI444P10LE.m in Sources */,
 				E654EAA91B6B283D00B0F2D0 /* IJKAVMoviePlayerController.m in Sources */,
 				E654EAAC1B6B284C00B0F2D0 /* IJKFFMoviePlayerDef.m in Sources */,
 				E654EAB91B6B286700B0F2D0 /* ijkplayer_ios.m in Sources */,

ios/IJKMediaPlayer/IJKMediaPlayer/ijkmedia/ijkplayer/ios/pipeline/IJKVideoToolBox.m

@@ -832,7 +832,7 @@ VideoToolBoxContext* init_videotoolbox(FFPlayer* ffp, AVCodecContext* ic)
 
     context_vtb->idr_based_identified = true;
     context_vtb->ffp = ffp;
-
+#if 0
     switch (profile) {
         case FF_PROFILE_H264_HIGH_10:
             if ([IJKDeviceModel currentModel].rank >= kIJKDeviceRank_AppleA7Class) {
@@ -848,7 +848,7 @@ VideoToolBoxContext* init_videotoolbox(FFPlayer* ffp, AVCodecContext* ic)
         case FF_PROFILE_H264_CAVLC_444:
             goto failed;
     }
-
+#endif
     if (width < 0 || height < 0) {
         goto failed;
     }

ios/IJKMediaPlayer/IJKMediaPlayer/ijkmedia/ijksdl/ios/IJKSDLGLRenderI444P10LE.h

+/*
+ * IJKSDLGLRenderI444P10LE.h
+ *
+ * Copyright (c) 2015 Zhang Rui <bbcallen@gmail.com>
+ *
+ * based on https://github.com/kolyvan/kxmovie
+ *
+ * This file is part of ijkPlayer.
+ *
+ * ijkPlayer is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * ijkPlayer is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with ijkPlayer; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#import <Foundation/Foundation.h>
+#import "IJKSDLGLRender.h"
+
+@interface IJKSDLGLRenderI444P10LE : NSObject<IJKSDLGLRender>
+
+@end

ios/IJKMediaPlayer/IJKMediaPlayer/ijkmedia/ijksdl/ios/IJKSDLGLRenderI444P10LE.m

+/*
+ * IJKSDLGLRenderI444P10LE.m
+ *
+ * Copyright (c) 2015 Zhang Rui <bbcallen@gmail.com>
+ *
+ * based on https://github.com/kolyvan/kxmovie
+ *
+ * This file is part of ijkPlayer.
+ *
+ * ijkPlayer is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * ijkPlayer is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with ijkPlayer; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#import "IJKSDLGLRenderI444P10LE.h"
+#import "IJKSDLGLShader.h"
+
+static NSString *const g_yuvFragmentShaderString = IJK_SHADER_STRING
+(
+    varying highp vec2 v_texcoord;
+    precision mediump float;
+    uniform sampler2D SamplerY;
+    uniform sampler2D SamplerU;
+    uniform sampler2D SamplerV;
+    uniform mat3 colorConversionMatrix;
+
+    void main()
+    {
+        highp vec3 yuvr;
+        highp vec3 yuvg;
+        highp vec3 yuv;
+        lowp vec3 rgb;
+
+        // Subtract constants to map the video range start at 0
+        yuvr.x = texture2D(SamplerY, v_texcoord).r;
+        yuvr.y = texture2D(SamplerU, v_texcoord).r;
+        yuvr.z = texture2D(SamplerV, v_texcoord).r;
+        yuvg.x = texture2D(SamplerY, v_texcoord).g;
+        yuvg.y = texture2D(SamplerU, v_texcoord).g;
+        yuvg.z = texture2D(SamplerV, v_texcoord).g;
+
+        yuv    = (yuvr * 255.0 + yuvg * 255.0 * 256.0) / (1023.0);
+        yuv.x  = yuv.x - 16.0 / 255.0;
+        yuv.yz = yuv.yz - vec2(0.5, 0.5);
+
+        rgb = colorConversionMatrix * yuv;
+        gl_FragColor = vec4(rgb,1);
+    }
+);
+
+@implementation IJKSDLGLRenderI444P10LE {
+    GLint _uniform[1];
+    GLint _uniformSamplers[3];
+    GLuint _textures[3];
+
+    const GLfloat *_preferredConversion;
+}
+
+- (BOOL) isValid
+{
+    return (_textures[0] != 0);
+}
+
+- (NSString *) fragmentShader
+{
+    return g_yuvFragmentShaderString;
+}
+
+- (void) resolveUniforms: (GLuint) program
+{
+    _uniformSamplers[0] = glGetUniformLocation(program, "SamplerY");
+    _uniformSamplers[1] = glGetUniformLocation(program, "SamplerU");
+    _uniformSamplers[2] = glGetUniformLocation(program, "SamplerV");
+    _uniform[0] = glGetUniformLocation(program, "colorConversionMatrix");
+}
+
+- (void) render: (SDL_VoutOverlay *) overlay
+{
+    assert(overlay->planes);
+    assert(overlay->format == SDL_FCC_I444P10LE);
+    assert(overlay->planes == 3);
+    // assert(yuvFrame.luma.length == yuvFrame.width * yuvFrame.height);
+    // assert(yuvFrame.chromaB.length == (yuvFrame.width * yuvFrame.height) / 4);
+    // assert(yuvFrame.chromaR.length == (yuvFrame.width * yuvFrame.height) / 4);
+
+    const NSUInteger frameHeight = overlay->h;
+
+    glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
+
+    if (0 == _textures[0])
+        glGenTextures(3, _textures);
+
+    _preferredConversion = kColorConversion709;
+
+    const UInt8 *pixels[3] = { overlay->pixels[0], overlay->pixels[1], overlay->pixels[2] };
+    const NSUInteger widths[3]  = { overlay->pitches[0] / 2, overlay->pitches[1] / 2, overlay->pitches[2] / 2 };
+    const NSUInteger heights[3] = { frameHeight, frameHeight, frameHeight };
+
+    for (int i = 0; i < 3; ++i) {
+
+        glBindTexture(GL_TEXTURE_2D, _textures[i]);
+
+        glTexImage2D(GL_TEXTURE_2D,
+                     0,
+                     GL_RG_EXT,
+                     (int)widths[i],
+                     (int)heights[i],
+                     0,
+                     GL_RG_EXT,
+                     GL_UNSIGNED_BYTE,
+                     pixels[i]);
+
+        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
+        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
+        glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+        glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+    }
+}
+
+- (BOOL) prepareDisplay
+{
+    if (_textures[0] == 0)
+        return NO;
+
+    for (int i = 0; i < 3; ++i) {
+        glActiveTexture(GL_TEXTURE0 + i);
+        glBindTexture(GL_TEXTURE_2D, _textures[i]);
+        glUniform1i(_uniformSamplers[i], i);
+    }
+
+    glUniformMatrix3fv(_uniform[0], 1, GL_FALSE, _preferredConversion);
+    return YES;
+}
+
+- (void) dealloc
+{
+    if (_textures[0])
+        glDeleteTextures(3, _textures);
+}
+
+@end

ios/IJKMediaPlayer/IJKMediaPlayer/ijkmedia/ijksdl/ios/IJKSDLGLView.m

@@ -28,6 +28,7 @@
 #import "IJKSDLGLRenderI420.h"
 #import "IJKSDLGLRenderRV24.h"
 #import "IJKSDLGLRenderNV12.h"
+#import "IJKSDLGLRenderI444P10LE.h"
 #include "ijksdl/ijksdl_timer.h"
 #include "ijksdl/ios/ijksdl_ios.h"
 #import "IJKSDLHudViewController.h"
@@ -489,6 +490,11 @@ static int g_ijk_gles_queue_spec_key;
             _renderer = [[IJKSDLGLRenderRV24 alloc] init];
             _bytesPerPixel = 3;
             NSLog(@"OK use RV24 GL renderer");
+        } else if (overlay->format == SDL_FCC_I444P10LE) {
+            _frameChroma = overlay->format;
+            _renderer = [[IJKSDLGLRenderI444P10LE alloc] init];
+            _bytesPerPixel = 2;
+            NSLog(@"OK use I444 GL renderer");
         }
 
         if (![self loadShaders]) {