// Copyright 2013 The Flutter Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. #import "flutter/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.h" #include "flutter/fml/logging.h" #import "flutter/shell/platform/darwin/common/framework/Headers/FlutterMacros.h" #include "third_party/skia/include/core/SkYUVAInfo.h" #include "third_party/skia/include/gpu/GrBackendSurface.h" #include "third_party/skia/include/gpu/GrDirectContext.h" #include "third_party/skia/include/gpu/GrYUVABackendTextures.h" #include "third_party/skia/include/gpu/mtl/GrMtlTypes.h" FLUTTER_ASSERT_ARC namespace { static sk_cf_obj SkiaTextureFromCVMetalTexture(CVMetalTextureRef cvMetalTexture) { id texture = CVMetalTextureGetTexture(cvMetalTexture); // CVMetal texture can be released as soon as we can the MTLTexture from it. CVPixelBufferRelease(cvMetalTexture); return sk_cf_obj{(__bridge_retained const void*)texture}; } } @implementation FlutterDarwinExternalTextureMetal { CVMetalTextureCacheRef _textureCache; NSObject* _externalTexture; BOOL _textureFrameAvailable; sk_sp _externalImage; CVPixelBufferRef _lastPixelBuffer; OSType _pixelFormat; } - (instancetype)initWithTextureCache:(nonnull CVMetalTextureCacheRef)textureCache textureID:(int64_t)textureID texture:(NSObject*)texture { if (self = [super init]) { _textureCache = textureCache; CFRetain(_textureCache); _textureID = textureID; _externalTexture = texture; return self; } return nil; } - (void)dealloc { CVPixelBufferRelease(_lastPixelBuffer); if (_textureCache) { CFRelease(_textureCache); } } - (void)paint:(SkCanvas&)canvas bounds:(const SkRect&)bounds freeze:(BOOL)freeze grContext:(nonnull GrDirectContext*)grContext sampling:(const SkSamplingOptions&)sampling { const bool needsUpdatedTexture = (!freeze && _textureFrameAvailable) || !_externalImage; if (needsUpdatedTexture) { [self onNeedsUpdatedTexture:grContext]; } if (_externalImage) { canvas.drawImageRect(_externalImage, // image SkRect::Make(_externalImage->bounds()), // source rect bounds, // destination rect sampling, // sampling nullptr, // paint SkCanvas::SrcRectConstraint::kFast_SrcRectConstraint // constraint ); } } - (void)onNeedsUpdatedTexture:(nonnull GrDirectContext*)grContext { CVPixelBufferRef pixelBuffer = [_externalTexture copyPixelBuffer]; if (pixelBuffer) { CVPixelBufferRelease(_lastPixelBuffer); _lastPixelBuffer = pixelBuffer; _pixelFormat = CVPixelBufferGetPixelFormatType(_lastPixelBuffer); } // If the application told us there was a texture frame available but did not provide one when // asked for it, reuse the previous texture but make sure to ask again the next time around. sk_sp image = [self wrapExternalPixelBuffer:_lastPixelBuffer grContext:grContext]; if (image) { _externalImage = image; _textureFrameAvailable = false; } } - (void)onGrContextCreated { // External images in this backend have no thread affinity and are not tied to the context in any // way. Instead, they are tied to the Metal device which is associated with the cache already and // is consistent throughout the shell run. } - (void)onGrContextDestroyed { // The image must be reset because it is tied to the onscreen context. But the pixel buffer that // created the image is still around. In case of context reacquisition, that last pixel // buffer will be used to materialize the image in case the application fails to provide a new // one. _externalImage.reset(); CVMetalTextureCacheFlush(_textureCache, // cache 0 // options (must be zero) ); } - (void)markNewFrameAvailable { _textureFrameAvailable = YES; } - (void)onTextureUnregistered { if ([_externalTexture respondsToSelector:@selector(onTextureUnregistered:)]) { [_externalTexture onTextureUnregistered:_externalTexture]; } } #pragma mark - External texture skia wrapper methods. - (sk_sp)wrapExternalPixelBuffer:(CVPixelBufferRef)pixelBuffer grContext:(GrDirectContext*)grContext { if (!pixelBuffer) { return nullptr; } sk_sp image = nullptr; if (_pixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange || _pixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) { image = [self wrapNV12ExternalPixelBuffer:pixelBuffer grContext:grContext]; } else { image = [self wrapRGBAExternalPixelBuffer:pixelBuffer grContext:grContext]; } if (!image) { FML_DLOG(ERROR) << "Could not wrap Metal texture as a Skia image."; } return image; } - (sk_sp)wrapNV12ExternalPixelBuffer:(CVPixelBufferRef)pixelBuffer grContext:(GrDirectContext*)grContext { SkISize textureSize = SkISize::Make(CVPixelBufferGetWidth(pixelBuffer), CVPixelBufferGetHeight(pixelBuffer)); CVMetalTextureRef yMetalTexture = nullptr; { CVReturn cvReturn = CVMetalTextureCacheCreateTextureFromImage(/*allocator=*/kCFAllocatorDefault, /*textureCache=*/_textureCache, /*sourceImage=*/pixelBuffer, /*textureAttributes=*/nullptr, /*pixelFormat=*/MTLPixelFormatR8Unorm, /*width=*/textureSize.width(), /*height=*/textureSize.height(), /*planeIndex=*/0u, /*texture=*/&yMetalTexture); if (cvReturn != kCVReturnSuccess) { FML_DLOG(ERROR) << "Could not create Metal texture from pixel buffer: CVReturn " << cvReturn; return nullptr; } } CVMetalTextureRef uvMetalTexture = nullptr; { CVReturn cvReturn = CVMetalTextureCacheCreateTextureFromImage(/*allocator=*/kCFAllocatorDefault, /*textureCache=*/_textureCache, /*sourceImage=*/pixelBuffer, /*textureAttributes=*/nullptr, /*pixelFormat=*/MTLPixelFormatRG8Unorm, /*width=*/textureSize.width() / 2, /*height=*/textureSize.height() / 2, /*planeIndex=*/1u, /*texture=*/&uvMetalTexture); if (cvReturn != kCVReturnSuccess) { FML_DLOG(ERROR) << "Could not create Metal texture from pixel buffer: CVReturn " << cvReturn; return nullptr; } } GrMtlTextureInfo ySkiaTextureInfo; ySkiaTextureInfo.fTexture = SkiaTextureFromCVMetalTexture(yMetalTexture); GrBackendTexture skiaBackendTextures[2]; skiaBackendTextures[0] = GrBackendTexture(/*width=*/textureSize.width(), /*height=*/textureSize.height(), /*mipMapped=*/GrMipMapped ::kNo, /*textureInfo=*/ySkiaTextureInfo); GrMtlTextureInfo uvSkiaTextureInfo; uvSkiaTextureInfo.fTexture = SkiaTextureFromCVMetalTexture(uvMetalTexture); skiaBackendTextures[1] = GrBackendTexture(/*width=*/textureSize.width(), /*height=*/textureSize.height(), /*mipMapped=*/GrMipMapped ::kNo, /*textureInfo=*/uvSkiaTextureInfo); SkYUVAInfo yuvaInfo(skiaBackendTextures[0].dimensions(), SkYUVAInfo::PlaneConfig::kY_UV, SkYUVAInfo::Subsampling::k444, kRec601_SkYUVColorSpace); GrYUVABackendTextures yuvaBackendTextures(yuvaInfo, skiaBackendTextures, kTopLeft_GrSurfaceOrigin); sk_sp image = SkImage::MakeFromYUVATextures(grContext, yuvaBackendTextures, /*imageColorSpace=*/nullptr, /*releaseProc*/ nullptr, /*releaseContext*/ nullptr); return image; } - (sk_sp)wrapRGBAExternalPixelBuffer:(CVPixelBufferRef)pixelBuffer grContext:(GrDirectContext*)grContext { SkISize textureSize = SkISize::Make(CVPixelBufferGetWidth(pixelBuffer), CVPixelBufferGetHeight(pixelBuffer)); CVMetalTextureRef metalTexture = nullptr; CVReturn cvReturn = CVMetalTextureCacheCreateTextureFromImage(/*allocator=*/kCFAllocatorDefault, /*textureCache=*/_textureCache, /*sourceImage=*/pixelBuffer, /*textureAttributes=*/nullptr, /*pixelFormat=*/MTLPixelFormatBGRA8Unorm, /*width=*/textureSize.width(), /*height=*/textureSize.height(), /*planeIndex=*/0u, /*texture=*/&metalTexture); if (cvReturn != kCVReturnSuccess) { FML_DLOG(ERROR) << "Could not create Metal texture from pixel buffer: CVReturn " << cvReturn; return nullptr; } GrMtlTextureInfo skiaTextureInfo; skiaTextureInfo.fTexture = SkiaTextureFromCVMetalTexture(metalTexture); GrBackendTexture skiaBackendTexture(/*width=*/textureSize.width(), /*height=*/textureSize.height(), /*mipMapped=*/GrMipMapped ::kNo, /*textureInfo=*/skiaTextureInfo); sk_sp image = SkImage::MakeFromTexture(grContext, skiaBackendTexture, kTopLeft_GrSurfaceOrigin, kBGRA_8888_SkColorType, kPremul_SkAlphaType, /*imageColorSpace=*/nullptr, /*releaseProc*/ nullptr, /*releaseContext*/ nullptr ); return image; } @end