Allow external texture sources when using the Metal backend. (#17154)

ci/licenses_golden/licenses_flutter

@@ -189,6 +189,7 @@ FILE: ../../../flutter/fml/platform/android/scoped_java_ref.cc
 FILE: ../../../flutter/fml/platform/android/scoped_java_ref.h
 FILE: ../../../flutter/fml/platform/darwin/cf_utils.cc
 FILE: ../../../flutter/fml/platform/darwin/cf_utils.h
+FILE: ../../../flutter/fml/platform/darwin/cf_utils_unittests.mm
 FILE: ../../../flutter/fml/platform/darwin/message_loop_darwin.h
 FILE: ../../../flutter/fml/platform/darwin/message_loop_darwin.mm
 FILE: ../../../flutter/fml/platform/darwin/paths_darwin.mm
@@ -890,6 +891,8 @@ FILE: ../../../flutter/shell/platform/darwin/ios/ios_context_software.h
 FILE: ../../../flutter/shell/platform/darwin/ios/ios_context_software.mm
 FILE: ../../../flutter/shell/platform/darwin/ios/ios_external_texture_gl.h
 FILE: ../../../flutter/shell/platform/darwin/ios/ios_external_texture_gl.mm
+FILE: ../../../flutter/shell/platform/darwin/ios/ios_external_texture_metal.h
+FILE: ../../../flutter/shell/platform/darwin/ios/ios_external_texture_metal.mm
 FILE: ../../../flutter/shell/platform/darwin/ios/ios_render_target_gl.h
 FILE: ../../../flutter/shell/platform/darwin/ios/ios_render_target_gl.mm
 FILE: ../../../flutter/shell/platform/darwin/ios/ios_surface.h

flow/texture.cc

@@ -13,12 +13,19 @@ Texture::~Texture() = default;
 TextureRegistry::TextureRegistry() = default;
 
 void TextureRegistry::RegisterTexture(std::shared_ptr<Texture> texture) {
+  if (!texture) {
+    return;
+  }
   mapping_[texture->Id()] = texture;
 }
 
 void TextureRegistry::UnregisterTexture(int64_t id) {
-  mapping_[id]->OnTextureUnregistered();
-  mapping_.erase(id);
+  auto found = mapping_.find(id);
+  if (found == mapping_.end()) {
+    return;
+  }
+  found->second->OnTextureUnregistered();
+  mapping_.erase(found);
 }
 
 void TextureRegistry::OnGrContextCreated() {

fml/BUILD.gn

@@ -252,7 +252,6 @@ executable("fml_unittests") {
     "message_loop_unittests.cc",
     "message_unittests.cc",
     "paths_unittests.cc",
-    "platform/darwin/string_range_sanitization_unittests.mm",
     "synchronization/count_down_latch_unittests.cc",
     "synchronization/semaphore_unittest.cc",
     "synchronization/sync_switch_unittest.cc",
@@ -264,6 +263,13 @@ executable("fml_unittests") {
     "time/time_unittest.cc",
   ]
 
+  if (is_mac) {
+    sources += [
+      "platform/darwin/cf_utils_unittests.mm",
+      "platform/darwin/string_range_sanitization_unittests.mm",
+    ]
+  }
+
   deps = [
     ":fml_fixtures",
     "//flutter/fml",

fml/platform/darwin/cf_utils.h

@@ -18,6 +18,21 @@ class CFRef {
 
   CFRef(T instance) : instance_(instance) {}
 
+  CFRef(const CFRef& other) : instance_(other.instance_) {
+    if (instance_) {
+      CFRetain(instance_);
+    }
+  }
+
+  CFRef(CFRef&& other) : instance_(other.instance_) {
+    other.instance_ = nullptr;
+  }
+
+  CFRef& operator=(CFRef&& other) {
+    Reset(other.Release());
+    return *this;
+  }
+
   ~CFRef() {
     if (instance_ != nullptr) {
       CFRelease(instance_);
@@ -25,7 +40,7 @@ class CFRef {
     instance_ = nullptr;
   }
 
-  void Reset(T instance) {
+  void Reset(T instance = nullptr) {
     if (instance_ == instance) {
       return;
     }
@@ -36,6 +51,12 @@ class CFRef {
     instance_ = instance;
   }
 
+  [[nodiscard]] T Release() {
+    auto instance = instance_;
+    instance_ = nullptr;
+    return instance;
+  }
+
   operator T() const { return instance_; }
 
   operator bool() const { return instance_ != nullptr; }
@@ -43,7 +64,7 @@ class CFRef {
  private:
   T instance_;
 
-  FML_DISALLOW_COPY_AND_ASSIGN(CFRef);
+  CFRef& operator=(const CFRef&) = delete;
 };
 
 }  // namespace fml

fml/platform/darwin/cf_utils_unittests.mm

+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "flutter/fml/platform/darwin/cf_utils.h"
+#include "flutter/testing/testing.h"
+
+namespace fml {
+namespace testing {
+
+TEST(CFTest, CanCreateRefs) {
+  CFRef<CFMutableStringRef> string(CFStringCreateMutable(kCFAllocatorDefault, 100u));
+  // Cast
+  ASSERT_TRUE(static_cast<bool>(string));
+  ASSERT_TRUE(string);
+
+  const auto ref_count = CFGetRetainCount(string);
+
+  // Copy & Reset
+  {
+    CFRef<CFMutableStringRef> string2 = string;
+    ASSERT_TRUE(string2);
+    ASSERT_EQ(ref_count + 1u, CFGetRetainCount(string));
+    ASSERT_EQ(CFGetRetainCount(string2), CFGetRetainCount(string));
+
+    string2.Reset();
+    ASSERT_FALSE(string2);
+    ASSERT_EQ(ref_count, CFGetRetainCount(string));
+  }
+
+  // Release
+  {
+    auto string3 = string;
+    ASSERT_TRUE(string3);
+    ASSERT_EQ(ref_count + 1u, CFGetRetainCount(string));
+    auto raw_string3 = string3.Release();
+    ASSERT_FALSE(string3);
+    ASSERT_EQ(ref_count + 1u, CFGetRetainCount(string));
+    CFRelease(raw_string3);
+    ASSERT_EQ(ref_count, CFGetRetainCount(string));
+  }
+
+  // Move
+  {
+    auto string_source = string;
+    ASSERT_TRUE(string_source);
+    auto string_move = std::move(string_source);
+    ASSERT_FALSE(string_source);
+    ASSERT_EQ(ref_count + 1u, CFGetRetainCount(string));
+    string_move.Reset();
+    ASSERT_EQ(ref_count, CFGetRetainCount(string));
+  }
+
+  // Move assign.
+  {
+    auto string_move_assign = std::move(string);
+    ASSERT_FALSE(string);
+    ASSERT_EQ(ref_count, CFGetRetainCount(string_move_assign));
+  }
+}
+
+}  // namespace testing
+}  // namespace fml

shell/platform/darwin/ios/BUILD.gn

@@ -131,6 +131,8 @@ shared_library("create_flutter_framework_dylib") {
     sources += [
       "ios_context_metal.h",
       "ios_context_metal.mm",
+      "ios_external_texture_metal.h",
+      "ios_external_texture_metal.mm",
       "ios_surface_metal.h",
       "ios_surface_metal.mm",
     ]

shell/platform/darwin/ios/ios_context.h

@@ -7,8 +7,10 @@
 
 #include <memory>
 
+#include "flutter/flow/texture.h"
 #include "flutter/fml/macros.h"
 #include "flutter/fml/platform/darwin/scoped_nsobject.h"
+#include "flutter/shell/platform/darwin/common/framework/Headers/FlutterTexture.h"
 #include "flutter/shell/platform/darwin/ios/rendering_api_selection.h"
 #include "third_party/skia/include/gpu/GrContext.h"
 
@@ -119,6 +121,20 @@ class IOSContext {
   ///
   virtual bool ClearCurrent() = 0;
 
+  //----------------------------------------------------------------------------
+  /// @brief      Creates an external texture proxy of the appropriate client
+  ///             rendering API.
+  ///
+  /// @param[in]  texture_id  The texture identifier
+  /// @param[in]  texture     The texture
+  ///
+  /// @return     The texture proxy if the rendering backend supports embedder
+  ///             provided external textures.
+  ///
+  virtual std::unique_ptr<Texture> CreateExternalTexture(
+      int64_t texture_id,
+      fml::scoped_nsobject<NSObject<FlutterTexture>> texture) = 0;
+
  protected:
   IOSContext();
 

shell/platform/darwin/ios/ios_context_gl.h

@@ -40,6 +40,11 @@ class IOSContextGL final : public IOSContext {
   // |IOSContext|
   bool ResourceMakeCurrent() override;
 
+  // |IOSContext|
+  std::unique_ptr<Texture> CreateExternalTexture(
+      int64_t texture_id,
+      fml::scoped_nsobject<NSObject<FlutterTexture>> texture) override;
+
   FML_DISALLOW_COPY_AND_ASSIGN(IOSContextGL);
 };
 

shell/platform/darwin/ios/ios_context_gl.mm

@@ -8,6 +8,7 @@
 
 #include "flutter/shell/common/shell_io_manager.h"
 #include "flutter/shell/gpu/gpu_surface_gl_delegate.h"
+#include "flutter/shell/platform/darwin/ios/ios_external_texture_gl.h"
 
 namespace flutter {
 
@@ -58,4 +59,11 @@ bool IOSContextGL::ClearCurrent() {
   return [EAGLContext setCurrentContext:nil];
 }
 
+// |IOSContext|
+std::unique_ptr<Texture> IOSContextGL::CreateExternalTexture(
+    int64_t texture_id,
+    fml::scoped_nsobject<NSObject<FlutterTexture>> texture) {
+  return std::make_unique<IOSExternalTextureGL>(texture_id, std::move(texture));
+}
+
 }  // namespace flutter

shell/platform/darwin/ios/ios_context_metal.h

@@ -8,6 +8,7 @@
 #include <Metal/Metal.h>
 
 #include "flutter/fml/macros.h"
+#include "flutter/fml/platform/darwin/cf_utils.h"
 #include "flutter/fml/platform/darwin/scoped_nsobject.h"
 #include "flutter/shell/platform/darwin/ios/ios_context.h"
 #include "third_party/skia/include/gpu/GrContext.h"
@@ -35,6 +36,7 @@ class IOSContextMetal final : public IOSContext {
   fml::scoped_nsprotocol<id<MTLCommandQueue>> main_queue_;
   sk_sp<GrContext> main_context_;
   sk_sp<GrContext> resource_context_;
+  fml::CFRef<CVMetalTextureCacheRef> texture_cache_;
   bool is_valid_ = false;
 
   // |IOSContext|
@@ -49,6 +51,11 @@ class IOSContextMetal final : public IOSContext {
   // |IOSContext|
   bool ClearCurrent() override;
 
+  // |IOSContext|
+  std::unique_ptr<Texture> CreateExternalTexture(
+      int64_t texture_id,
+      fml::scoped_nsobject<NSObject<FlutterTexture>> texture) override;
+
   FML_DISALLOW_COPY_AND_ASSIGN(IOSContextMetal);
 };
 

shell/platform/darwin/ios/ios_context_metal.mm

@@ -5,20 +5,21 @@
 #include "flutter/shell/platform/darwin/ios/ios_context_metal.h"
 
 #include "flutter/fml/logging.h"
+#include "flutter/shell/platform/darwin/ios/ios_external_texture_metal.h"
 
 namespace flutter {
 
 IOSContextMetal::IOSContextMetal() {
   device_.reset([MTLCreateSystemDefaultDevice() retain]);
   if (!device_) {
-    FML_LOG(ERROR) << "Could not acquire Metal device.";
+    FML_DLOG(ERROR) << "Could not acquire Metal device.";
     return;
   }
 
   main_queue_.reset([device_ newCommandQueue]);
 
   if (!main_queue_) {
-    FML_LOG(ERROR) << "Could not create Metal command queue.";
+    FML_DLOG(ERROR) << "Could not create Metal command queue.";
     return;
   }
 
@@ -30,10 +31,23 @@ IOSContextMetal::IOSContextMetal() {
   resource_context_ = GrContext::MakeMetal([device_ retain], [main_queue_ retain]);
 
   if (!main_context_ || !resource_context_) {
-    FML_LOG(ERROR) << "Could not create Skia Metal contexts.";
+    FML_DLOG(ERROR) << "Could not create Skia Metal contexts.";
     return;
   }
 
+  CVMetalTextureCacheRef texture_cache_raw = NULL;
+  auto cv_return = CVMetalTextureCacheCreate(kCFAllocatorDefault,  // allocator
+                                             NULL,           // cache attributes (NULL default)
+                                             device_.get(),  // metal device
+                                             NULL,           // texture attributes (NULL default)
+                                             &texture_cache_raw  // [out] cache
+  );
+  if (cv_return != kCVReturnSuccess) {
+    FML_DLOG(ERROR) << "Could not create Metal texture cache.";
+    return;
+  }
+  texture_cache_.Reset(texture_cache_raw);
+
   is_valid_ = false;
 }
 
@@ -83,4 +97,11 @@ bool IOSContextMetal::ClearCurrent() {
   return true;
 }
 
+// |IOSContext|
+std::unique_ptr<Texture> IOSContextMetal::CreateExternalTexture(
+    int64_t texture_id,
+    fml::scoped_nsobject<NSObject<FlutterTexture>> texture) {
+  return std::make_unique<IOSExternalTextureMetal>(texture_id, texture_cache_, std::move(texture));
+}
+
 }  // namespace flutter

shell/platform/darwin/ios/ios_context_software.h

@@ -29,6 +29,11 @@ class IOSContextSoftware final : public IOSContext {
   // |IOSContext|
   bool ClearCurrent() override;
 
+  // |IOSContext|
+  std::unique_ptr<Texture> CreateExternalTexture(
+      int64_t texture_id,
+      fml::scoped_nsobject<NSObject<FlutterTexture>> texture) override;
+
  private:
   FML_DISALLOW_COPY_AND_ASSIGN(IOSContextSoftware);
 };

shell/platform/darwin/ios/ios_context_software.mm

@@ -31,4 +31,18 @@ bool IOSContextSoftware::ClearCurrent() {
   return false;
 }
 
+// |IOSContext|
+std::unique_ptr<Texture> IOSContextSoftware::CreateExternalTexture(
+    int64_t texture_id,
+    fml::scoped_nsobject<NSObject<FlutterTexture>> texture) {
+  // Don't use FML for logging as it will contain engine specific details. This is a user facing
+  // message.
+  NSLog(@"Flutter: Attempted to composite external texture sources using the software backend. "
+        @"This backend is only used on simulators. This feature is only available on actual "
+        @"devices where OpenGL or Metal is used for rendering.");
+
+  // Not supported in this backend.
+  return nullptr;
+}
+
 }  // namespace flutter

shell/platform/darwin/ios/ios_external_texture_gl.mm

@@ -56,7 +56,7 @@ void IOSExternalTextureGL::CreateTextureFromPixelBuffer() {
 
 bool IOSExternalTextureGL::NeedUpdateTexture(bool freeze) {
   // Update texture if `texture_ref_` is reset to `nullptr` when GrContext
-  // is destroied or new frame is ready.
+  // is destroyed or new frame is ready.
   return (!freeze && new_frame_ready_) || !texture_ref_;
 }
 

shell/platform/darwin/ios/ios_external_texture_metal.h

+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef FLUTTER_SHELL_PLATFORM_DARWIN_IOS_IOS_EXTERNAL_TEXTURE_METAL_H_
+#define FLUTTER_SHELL_PLATFORM_DARWIN_IOS_IOS_EXTERNAL_TEXTURE_METAL_H_
+
+#include <atomic>
+
+#import <CoreVideo/CoreVideo.h>
+
+#include "flutter/flow/texture.h"
+#include "flutter/fml/macros.h"
+#include "flutter/fml/platform/darwin/cf_utils.h"
+#include "flutter/fml/platform/darwin/scoped_nsobject.h"
+#include "flutter/shell/platform/darwin/common/framework/Headers/FlutterTexture.h"
+#include "third_party/skia/include/core/SkImage.h"
+
+namespace flutter {
+
+class IOSExternalTextureMetal final : public Texture {
+ public:
+  IOSExternalTextureMetal(int64_t texture_id,
+                          fml::CFRef<CVMetalTextureCacheRef> texture_cache,
+                          fml::scoped_nsobject<NSObject<FlutterTexture>> external_texture);
+
+  // |Texture|
+  ~IOSExternalTextureMetal();
+
+ private:
+  fml::CFRef<CVMetalTextureCacheRef> texture_cache_;
+  fml::scoped_nsobject<NSObject<FlutterTexture>> external_texture_;
+  std::atomic_bool texture_frame_available_;
+  sk_sp<SkImage> external_image_;
+
+  // |Texture|
+  void Paint(SkCanvas& canvas, const SkRect& bounds, bool freeze, GrContext* context) override;
+
+  // |Texture|
+  void OnGrContextCreated() override;
+
+  // |Texture|
+  void OnGrContextDestroyed() override;
+
+  // |Texture|
+  void MarkNewFrameAvailable() override;
+
+  // |Texture|
+  void OnTextureUnregistered() override;
+
+  sk_sp<SkImage> WrapExternalPixelBuffer(GrContext* context);
+
+  FML_DISALLOW_COPY_AND_ASSIGN(IOSExternalTextureMetal);
+};
+
+}  // namespace flutter
+
+#endif  // FLUTTER_SHELL_PLATFORM_DARWIN_IOS_IOS_EXTERNAL_TEXTURE_METAL_H_

shell/platform/darwin/ios/ios_external_texture_metal.mm

+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "flutter/shell/platform/darwin/ios/ios_external_texture_metal.h"
+
+#include "flutter/fml/logging.h"
+#include "third_party/skia/include/gpu/GrBackendSurface.h"
+#include "third_party/skia/include/gpu/mtl/GrMtlTypes.h"
+
+namespace flutter {
+
+IOSExternalTextureMetal::IOSExternalTextureMetal(
+    int64_t texture_id,
+    fml::CFRef<CVMetalTextureCacheRef> texture_cache,
+    fml::scoped_nsobject<NSObject<FlutterTexture>> external_texture)
+    : Texture(texture_id),
+      texture_cache_(std::move(texture_cache)),
+      external_texture_(std::move(external_texture)) {
+  FML_DCHECK(texture_cache_);
+  FML_DCHECK(external_texture_);
+}
+
+IOSExternalTextureMetal::~IOSExternalTextureMetal() = default;
+
+void IOSExternalTextureMetal::Paint(SkCanvas& canvas,
+                                    const SkRect& bounds,
+                                    bool freeze,
+                                    GrContext* context) {
+  if (!freeze && texture_frame_available_) {
+    external_image_ = nullptr;
+  }
+
+  if (!external_image_) {
+    external_image_ = WrapExternalPixelBuffer(context);
+    texture_frame_available_ = false;
+  }
+
+  if (external_image_) {
+    canvas.drawImageRect(external_image_,                                      // image
+                         external_image_->bounds(),                            // source rect
+                         bounds,                                               // destination rect
+                         nullptr,                                              // paint
+                         SkCanvas::SrcRectConstraint::kFast_SrcRectConstraint  // constraint
+    );
+  }
+}
+
+sk_sp<SkImage> IOSExternalTextureMetal::WrapExternalPixelBuffer(GrContext* context) {
+  auto pixel_buffer = fml::CFRef<CVPixelBufferRef>([external_texture_ copyPixelBuffer]);
+  if (!pixel_buffer) {
+    return nullptr;
+  }
+
+  auto texture_size =
+      SkISize::Make(CVPixelBufferGetWidth(pixel_buffer), CVPixelBufferGetHeight(pixel_buffer));
+
+  CVMetalTextureRef metal_texture_raw = NULL;
+  auto cv_return =
+      CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault,       // allocator
+                                                texture_cache_,            // texture cache
+                                                pixel_buffer,              // source image
+                                                NULL,                      // texture attributes
+                                                MTLPixelFormatBGRA8Unorm,  // pixel format
+                                                texture_size.width(),      // width
+                                                texture_size.height(),     // height
+                                                0u,                        // plane index
+                                                &metal_texture_raw         // [out] texture
+      );
+
+  if (cv_return != kCVReturnSuccess) {
+    FML_DLOG(ERROR) << "Could not create Metal texture from pixel buffer: CVReturn " << cv_return;
+    return nullptr;
+  }
+
+  fml::CFRef<CVMetalTextureRef> metal_texture(metal_texture_raw);
+
+  GrMtlTextureInfo skia_texture_info;
+  skia_texture_info.fTexture = sk_cf_obj<const void*>{
+      [reinterpret_cast<NSObject*>(CVMetalTextureGetTexture(metal_texture)) retain]};
+
+  GrBackendTexture skia_backend_texture(texture_size.width(),   // width
+                                        texture_size.height(),  // height
+                                        GrMipMapped ::kNo,      // mip-mapped
+                                        skia_texture_info       // texture info
+  );
+
+  struct ImageCaptures {
+    fml::CFRef<CVPixelBufferRef> buffer;
+    fml::CFRef<CVMetalTextureRef> texture;
+  };
+
+  auto captures = std::make_unique<ImageCaptures>();
+  captures->buffer = std::move(pixel_buffer);
+  captures->texture = std::move(metal_texture);
+
+  SkImage::TextureReleaseProc release_proc = [](SkImage::ReleaseContext release_context) {
+    auto captures = reinterpret_cast<ImageCaptures*>(release_context);
+    delete captures;
+  };
+
+  auto image = SkImage::MakeFromTexture(context,                   // context
+                                        skia_backend_texture,      // backend texture
+                                        kTopLeft_GrSurfaceOrigin,  // origin
+                                        kBGRA_8888_SkColorType,    // color type
+                                        kPremul_SkAlphaType,       // alpha type
+                                        nullptr,                   // color space
+                                        release_proc,              // release proc
+                                        captures.release()         // release context
+
+  );
+
+  if (!image) {
+    FML_DLOG(ERROR) << "Could not wrap Metal texture as a Skia image.";
+  }
+
+  return image;
+}
+
+void IOSExternalTextureMetal::OnGrContextCreated() {
+  // External images in this backend have no thread affinity and are not tied to the context in any
+  // way. Instead, they are tied to the Metal device which is associated with the cache already and
+  // is consistent throughout the shell run.
+}
+
+void IOSExternalTextureMetal::OnGrContextDestroyed() {
+  external_image_.reset();
+  CVMetalTextureCacheFlush(texture_cache_,  // cache
+                           0                // options (must be zero)
+  );
+}
+
+void IOSExternalTextureMetal::MarkNewFrameAvailable() {
+  texture_frame_available_ = true;
+}
+
+void IOSExternalTextureMetal::OnTextureUnregistered() {
+  if ([external_texture_ respondsToSelector:@selector(onTextureUnregistered:)]) {
+    [external_texture_ onTextureUnregistered:external_texture_];
+  }
+}
+
+}  // namespace flutter

shell/platform/darwin/ios/platform_view_ios.mm

@@ -4,18 +4,14 @@
 
 #include "flutter/shell/platform/darwin/ios/platform_view_ios.h"
 
-#import <QuartzCore/CAEAGLLayer.h>
-
 #include <utility>
 
 #include "flutter/common/task_runners.h"
 #include "flutter/fml/synchronization/waitable_event.h"
 #include "flutter/fml/trace_event.h"
 #include "flutter/shell/common/shell_io_manager.h"
-#include "flutter/shell/gpu/gpu_surface_gl_delegate.h"
 #include "flutter/shell/platform/darwin/ios/framework/Source/FlutterViewController_Internal.h"
 #include "flutter/shell/platform/darwin/ios/framework/Source/vsync_waiter_ios.h"
-#include "flutter/shell/platform/darwin/ios/ios_external_texture_gl.h"
 
 namespace flutter {
 
@@ -92,7 +88,8 @@ PointerDataDispatcherMaker PlatformViewIOS::GetDispatcherMaker() {
 
 void PlatformViewIOS::RegisterExternalTexture(int64_t texture_id,
                                               NSObject<FlutterTexture>* texture) {
-  RegisterTexture(std::make_shared<IOSExternalTextureGL>(texture_id, texture));
+  RegisterTexture(ios_context_->CreateExternalTexture(
+      texture_id, fml::scoped_nsobject<NSObject<FlutterTexture>>{[texture retain]}));
 }
 
 // |PlatformView|