Enhance image_filter_layer caching to filter a cached child (#17175)

flow/layers/container_layer.cc

@@ -160,4 +160,36 @@ void ContainerLayer::UpdateSceneChildren(SceneUpdateContext& context) {
 
 #endif  // defined(OS_FUCHSIA)
 
+MergedContainerLayer::MergedContainerLayer() {
+  // Ensure the layer has only one direct child.
+  //
+  // Any children will actually be added as children of this empty
+  // ContainerLayer which can be accessed via ::GetContainerLayer().
+  // If only one child is ever added to this layer then that child
+  // will become the layer returned from ::GetCacheableChild().
+  // If multiple child layers are added, then this implicit container
+  // child becomes the cacheable child, but at the potential cost of
+  // not being as stable in the raster cache from frame to frame.
+  ContainerLayer::Add(std::make_shared<ContainerLayer>());
+}
+
+void MergedContainerLayer::Add(std::shared_ptr<Layer> layer) {
+  GetChildContainer()->Add(std::move(layer));
+}
+
+ContainerLayer* MergedContainerLayer::GetChildContainer() const {
+  FML_DCHECK(layers().size() == 1);
+
+  return static_cast<ContainerLayer*>(layers()[0].get());
+}
+
+Layer* MergedContainerLayer::GetCacheableChild() const {
+  ContainerLayer* child_container = GetChildContainer();
+  if (child_container->layers().size() == 1) {
+    return child_container->layers()[0].get();
+  }
+
+  return child_container;
+}
+
 }  // namespace flutter

flow/layers/container_layer.h

@@ -35,9 +35,6 @@ class ContainerLayer : public Layer {
   void UpdateSceneChildren(SceneUpdateContext& context);
 #endif  // defined(OS_FUCHSIA)
 
-  // For OpacityLayer to restructure to have a single child.
-  void ClearChildren() { layers_.clear(); }
-
   // Try to prepare the raster cache for a given layer.
   //
   // The raster cache would fail if either of the followings is true:
@@ -58,6 +55,81 @@ class ContainerLayer : public Layer {
   FML_DISALLOW_COPY_AND_ASSIGN(ContainerLayer);
 };
 
+//------------------------------------------------------------------------------
+/// Some ContainerLayer objects perform a rendering operation or filter on
+/// the rendered output of their children. Often that operation is changed
+/// slightly from frame to frame as part of an animation. During such an
+/// animation, the children can be cached if they are stable to avoid having
+/// to render them on every frame. Even if the children are not stable,
+/// rendering them into the raster cache during a Preroll operation will save
+/// an extra change of rendering surface during the Paint phase as compared
+/// to using the SaveLayer that would otherwise be needed with no caching.
+///
+/// Typically the Flutter Widget objects that lead to the creation of these
+/// layers will try to enforce only a single child Widget by their design.
+/// Unfortunately, the process of turning Widgets eventually into engine
+/// layers is not a 1:1 process so this layer might end up with multiple
+/// child layers even if the Widget only had a single child Widget.
+///
+/// When such a layer goes to cache the output of its children, it will
+/// need to supply a single layer to the cache mechanism since the raster
+/// cache uses a layer unique_id() as part of the cache key. If this layer
+/// ended up with multiple children, then it must first collect them into
+/// one layer for the cache mechanism. In order to provide a single layer
+/// for all of the children, this utility class will implicitly collect
+/// the children into a secondary ContainerLayer called the child container.
+///
+/// A by-product of creating a hidden child container, though, is that the
+/// child container is created new every time this layer is created with
+/// different properties, such as during an animation. In that scenario,
+/// it would be best to cache the single real child of this layer if it
+/// is unique and if it is stable from frame to frame. To facilitate this
+/// optimal caching strategy, this class implements two accessor methods
+/// to be used for different purposes:
+///
+/// When the layer needs to recurse to perform some operation on its children,
+/// it can call GetChildContainer() to return the hidden container containing
+/// all of the real children.
+///
+/// When the layer wants to cache the rendered contents of its children, it
+/// should call GetCacheableChild() for best performance. This method may
+/// end up returning the same layer as GetChildContainer(), but only if the
+/// conditions for optimal caching of a single child are not met.
+///
+class MergedContainerLayer : public ContainerLayer {
+ public:
+  MergedContainerLayer();
+
+  void Add(std::shared_ptr<Layer> layer) override;
+
+ protected:
+  /**
+   * @brief Returns the ContainerLayer used to hold all of the children of the
+   * MergedContainerLayer. Note that this may not be the best layer to use
+   * for caching the children.
+   *
+   * @see GetCacheableChild()
+   * @return the ContainerLayer child used to hold the children
+   */
+  ContainerLayer* GetChildContainer() const;
+
+  /**
+   * @brief Returns the best choice for a Layer object that can be used
+   * in RasterCache operations to cache the children.
+   *
+   * The returned Layer must represent all children and try to remain stable
+   * if the MergedContainerLayer is reconstructed in subsequent frames of
+   * the scene.
+   *
+   * @see GetChildContainer()
+   * @return the best candidate Layer for caching the children
+   */
+  Layer* GetCacheableChild() const;
+
+ private:
+  FML_DISALLOW_COPY_AND_ASSIGN(MergedContainerLayer);
+};
+
 }  // namespace flutter
 
 #endif  // FLUTTER_FLOW_LAYERS_CONTAINER_LAYER_H_

flow/layers/container_layer_unittests.cc

@@ -198,5 +198,75 @@ TEST_F(ContainerLayerTest, NeedsSystemComposite) {
                                                child_path2, child_paint2}}}));
 }
 
+TEST_F(ContainerLayerTest, MergedOneChild) {
+  SkPath child_path;
+  child_path.addRect(5.0f, 6.0f, 20.5f, 21.5f);
+  SkPaint child_paint(SkColors::kGreen);
+  SkMatrix initial_transform = SkMatrix::Translate(-0.5f, -0.5f);
+
+  auto mock_layer = std::make_shared<MockLayer>(child_path, child_paint);
+  auto layer = std::make_shared<MergedContainerLayer>();
+  layer->Add(mock_layer);
+
+  layer->Preroll(preroll_context(), initial_transform);
+  EXPECT_FALSE(preroll_context()->has_platform_view);
+  EXPECT_EQ(mock_layer->paint_bounds(), child_path.getBounds());
+  EXPECT_EQ(layer->paint_bounds(), child_path.getBounds());
+  EXPECT_TRUE(mock_layer->needs_painting());
+  EXPECT_TRUE(layer->needs_painting());
+  EXPECT_FALSE(mock_layer->needs_system_composite());
+  EXPECT_FALSE(layer->needs_system_composite());
+  EXPECT_EQ(mock_layer->parent_matrix(), initial_transform);
+  EXPECT_EQ(mock_layer->parent_cull_rect(), kGiantRect);
+
+  layer->Paint(paint_context());
+  EXPECT_EQ(mock_canvas().draw_calls(),
+            std::vector({MockCanvas::DrawCall{
+                0, MockCanvas::DrawPathData{child_path, child_paint}}}));
+}
+
+TEST_F(ContainerLayerTest, MergedMultipleChildren) {
+  SkPath child_path1;
+  child_path1.addRect(5.0f, 6.0f, 20.5f, 21.5f);
+  SkPath child_path2;
+  child_path2.addRect(58.0f, 2.0f, 16.5f, 14.5f);
+  SkPaint child_paint1(SkColors::kGray);
+  SkPaint child_paint2(SkColors::kGreen);
+  SkMatrix initial_transform = SkMatrix::Translate(-0.5f, -0.5f);
+
+  auto mock_layer1 = std::make_shared<MockLayer>(child_path1, child_paint1);
+  auto mock_layer2 = std::make_shared<MockLayer>(child_path2, child_paint2);
+  auto layer = std::make_shared<MergedContainerLayer>();
+  layer->Add(mock_layer1);
+  layer->Add(mock_layer2);
+
+  SkRect expected_total_bounds = child_path1.getBounds();
+  expected_total_bounds.join(child_path2.getBounds());
+  layer->Preroll(preroll_context(), initial_transform);
+  EXPECT_FALSE(preroll_context()->has_platform_view);
+  EXPECT_EQ(mock_layer1->paint_bounds(), child_path1.getBounds());
+  EXPECT_EQ(mock_layer2->paint_bounds(), child_path2.getBounds());
+  EXPECT_EQ(layer->paint_bounds(), expected_total_bounds);
+  EXPECT_TRUE(mock_layer1->needs_painting());
+  EXPECT_TRUE(mock_layer2->needs_painting());
+  EXPECT_TRUE(layer->needs_painting());
+  EXPECT_FALSE(mock_layer1->needs_system_composite());
+  EXPECT_FALSE(mock_layer2->needs_system_composite());
+  EXPECT_FALSE(layer->needs_system_composite());
+  EXPECT_EQ(mock_layer1->parent_matrix(), initial_transform);
+  EXPECT_EQ(mock_layer2->parent_matrix(), initial_transform);
+  EXPECT_EQ(mock_layer1->parent_cull_rect(), kGiantRect);
+  EXPECT_EQ(mock_layer2->parent_cull_rect(),
+            kGiantRect);  // Siblings are independent
+
+  layer->Paint(paint_context());
+  EXPECT_EQ(
+      mock_canvas().draw_calls(),
+      std::vector({MockCanvas::DrawCall{
+                       0, MockCanvas::DrawPathData{child_path1, child_paint1}},
+                   MockCanvas::DrawCall{0, MockCanvas::DrawPathData{
+                                               child_path2, child_paint2}}}));
+}
+
 }  // namespace testing
 }  // namespace flutter

flow/layers/image_filter_layer.cc

@@ -7,7 +7,9 @@
 namespace flutter {
 
 ImageFilterLayer::ImageFilterLayer(sk_sp<SkImageFilter> filter)
-    : filter_(std::move(filter)) {}
+    : filter_(std::move(filter)),
+      transformed_filter_(nullptr),
+      render_count_(1) {}
 
 void ImageFilterLayer::Preroll(PrerollContext* context,
                                const SkMatrix& matrix) {
@@ -16,28 +18,66 @@ void ImageFilterLayer::Preroll(PrerollContext* context,
   Layer::AutoPrerollSaveLayerState save =
       Layer::AutoPrerollSaveLayerState::Create(context);
 
-  child_paint_bounds_ = SkRect::MakeEmpty();
-  PrerollChildren(context, matrix, &child_paint_bounds_);
+  SkRect child_bounds = SkRect::MakeEmpty();
+  PrerollChildren(context, matrix, &child_bounds);
   if (filter_) {
-    const SkIRect filter_input_bounds = child_paint_bounds_.roundOut();
+    const SkIRect filter_input_bounds = child_bounds.roundOut();
     SkIRect filter_output_bounds =
         filter_->filterBounds(filter_input_bounds, SkMatrix::I(),
                               SkImageFilter::kForward_MapDirection);
-    set_paint_bounds(SkRect::Make(filter_output_bounds));
-  } else {
-    set_paint_bounds(child_paint_bounds_);
+    child_bounds = SkRect::Make(filter_output_bounds);
   }
+  set_paint_bounds(child_bounds);
+
+  transformed_filter_ = nullptr;
+  if (render_count_ >= kMinimumRendersBeforeCachingFilterLayer) {
+    // We have rendered this same ImageFilterLayer object enough
+    // times to consider its properties and children to be stable
+    // from frame to frame so we try to cache the layer itself
+    // for maximum performance.
+    TryToPrepareRasterCache(context, this, matrix);
+  } else {
+    // This ImageFilterLayer is not yet considered stable so we
+    // increment the count to measure how many times it has been
+    // seen from frame to frame.
+    render_count_++;
 
-  TryToPrepareRasterCache(context, this, matrix);
+    // Now we will try to pre-render the children into the cache.
+    // To apply the filter to pre-rendered children, we must first
+    // modify the filter to be aware of the transform under which
+    // the cached bitmap was produced. Some SkImageFilter
+    // instances can do this operation on some transforms and some
+    // (filters or transforms) cannot. We can only cache the children
+    // and apply the filter on the fly if this operation succeeds.
+    transformed_filter_ = filter_->makeWithLocalMatrix(matrix);
+    if (transformed_filter_) {
+      // With a modified SkImageFilter we can now try to cache the
+      // children to avoid their rendering costs if they remain
+      // stable between frames and also avoiding a rendering surface
+      // switch during the Paint phase even if they are not stable.
+      // This benefit is seen most during animations.
+      TryToPrepareRasterCache(context, GetCacheableChild(), matrix);
+    }
+  }
 }
 
 void ImageFilterLayer::Paint(PaintContext& context) const {
   TRACE_EVENT0("flutter", "ImageFilterLayer::Paint");
   FML_DCHECK(needs_painting());
 
-  if (context.raster_cache &&
-      context.raster_cache->Draw(this, *context.leaf_nodes_canvas)) {
-    return;
+  if (context.raster_cache) {
+    if (context.raster_cache->Draw(this, *context.leaf_nodes_canvas)) {
+      return;
+    }
+    if (transformed_filter_) {
+      SkPaint paint;
+      paint.setImageFilter(transformed_filter_);
+
+      if (context.raster_cache->Draw(GetCacheableChild(),
+                                     *context.leaf_nodes_canvas, &paint)) {
+        return;
+      }
+    }
   }
 
   SkPaint paint;
@@ -45,10 +85,10 @@ void ImageFilterLayer::Paint(PaintContext& context) const {
 
   // Normally a save_layer is sized to the current layer bounds, but in this
   // case the bounds of the child may not be the same as the filtered version
-  // so we use the child_paint_bounds_ which were snapshotted from the
-  // Preroll on the children before we adjusted them based on the filter.
-  Layer::AutoSaveLayer save_layer =
-      Layer::AutoSaveLayer::Create(context, child_paint_bounds_, &paint);
+  // so we use the bounds of the child container which do not include any
+  // modifications that the filter might apply.
+  Layer::AutoSaveLayer save_layer = Layer::AutoSaveLayer::Create(
+      context, GetChildContainer()->paint_bounds(), &paint);
   PaintChildren(context);
 }
 

flow/layers/image_filter_layer.h

@@ -11,7 +11,7 @@
 
 namespace flutter {
 
-class ImageFilterLayer : public ContainerLayer {
+class ImageFilterLayer : public MergedContainerLayer {
  public:
   ImageFilterLayer(sk_sp<SkImageFilter> filter);
 
@@ -20,8 +20,25 @@ class ImageFilterLayer : public ContainerLayer {
   void Paint(PaintContext& context) const override;
 
  private:
+  // The ImageFilterLayer might cache the filtered output of this layer
+  // if the layer remains stable (if it is not animating for instance).
+  // If the ImageFilterLayer is not the same between rendered frames,
+  // though, it will cache its children instead and filter their cached
+  // output on the fly.
+  // Caching just the children saves the time to render them and also
+  // avoids a rendering surface switch to draw them.
+  // Caching the layer itself avoids all of that and additionally avoids
+  // the cost of applying the filter, but can be worse than caching the
+  // children if the filter itself is not stable from frame to frame.
+  // This constant controls how many times we will Preroll and Paint this
+  // same ImageFilterLayer before we consider the layer and filter to be
+  // stable enough to switch from caching the children to caching the
+  // filtered output of this layer.
+  static constexpr int kMinimumRendersBeforeCachingFilterLayer = 3;
+
   sk_sp<SkImageFilter> filter_;
-  SkRect child_paint_bounds_;
+  sk_sp<SkImageFilter> transformed_filter_;
+  int render_count_;
 
   FML_DISALLOW_COPY_AND_ASSIGN(ImageFilterLayer);
 };

flow/layers/image_filter_layer_unittests.cc

@@ -260,5 +260,70 @@ TEST_F(ImageFilterLayerTest, Readback) {
   EXPECT_FALSE(preroll_context()->surface_needs_readback);
 }
 
+TEST_F(ImageFilterLayerTest, ChildIsCached) {
+  auto layer_filter = SkImageFilter::MakeMatrixFilter(
+      SkMatrix(), SkFilterQuality::kMedium_SkFilterQuality, nullptr);
+  auto initial_transform = SkMatrix::Translate(50.0, 25.5);
+  auto other_transform = SkMatrix::Scale(1.0, 2.0);
+  const SkPath child_path = SkPath().addRect(SkRect::MakeWH(5.0f, 5.0f));
+  auto mock_layer = std::make_shared<MockLayer>(child_path);
+  auto layer = std::make_shared<ImageFilterLayer>(layer_filter);
+  layer->Add(mock_layer);
+
+  SkMatrix cache_ctm = initial_transform;
+  SkCanvas cache_canvas;
+  cache_canvas.setMatrix(cache_ctm);
+  SkCanvas other_canvas;
+  other_canvas.setMatrix(other_transform);
+
+  use_mock_raster_cache();
+
+  EXPECT_EQ(raster_cache()->GetLayerCachedEntriesCount(), (size_t)0);
+  EXPECT_FALSE(raster_cache()->Draw(mock_layer.get(), other_canvas));
+  EXPECT_FALSE(raster_cache()->Draw(mock_layer.get(), cache_canvas));
+
+  layer->Preroll(preroll_context(), initial_transform);
+
+  EXPECT_EQ(raster_cache()->GetLayerCachedEntriesCount(), (size_t)1);
+  EXPECT_FALSE(raster_cache()->Draw(mock_layer.get(), other_canvas));
+  EXPECT_TRUE(raster_cache()->Draw(mock_layer.get(), cache_canvas));
+}
+
+TEST_F(ImageFilterLayerTest, ChildrenNotCached) {
+  auto layer_filter = SkImageFilter::MakeMatrixFilter(
+      SkMatrix(), SkFilterQuality::kMedium_SkFilterQuality, nullptr);
+  auto initial_transform = SkMatrix::Translate(50.0, 25.5);
+  auto other_transform = SkMatrix::Scale(1.0, 2.0);
+  const SkPath child_path1 = SkPath().addRect(SkRect::MakeWH(5.0f, 5.0f));
+  const SkPath child_path2 = SkPath().addRect(SkRect::MakeWH(5.0f, 5.0f));
+  auto mock_layer1 = std::make_shared<MockLayer>(child_path1);
+  auto mock_layer2 = std::make_shared<MockLayer>(child_path2);
+  auto layer = std::make_shared<ImageFilterLayer>(layer_filter);
+  layer->Add(mock_layer1);
+  layer->Add(mock_layer2);
+
+  SkMatrix cache_ctm = initial_transform;
+  SkCanvas cache_canvas;
+  cache_canvas.setMatrix(cache_ctm);
+  SkCanvas other_canvas;
+  other_canvas.setMatrix(other_transform);
+
+  use_mock_raster_cache();
+
+  EXPECT_EQ(raster_cache()->GetLayerCachedEntriesCount(), (size_t)0);
+  EXPECT_FALSE(raster_cache()->Draw(mock_layer1.get(), other_canvas));
+  EXPECT_FALSE(raster_cache()->Draw(mock_layer1.get(), cache_canvas));
+  EXPECT_FALSE(raster_cache()->Draw(mock_layer2.get(), other_canvas));
+  EXPECT_FALSE(raster_cache()->Draw(mock_layer2.get(), cache_canvas));
+
+  layer->Preroll(preroll_context(), initial_transform);
+
+  EXPECT_EQ(raster_cache()->GetLayerCachedEntriesCount(), (size_t)1);
+  EXPECT_FALSE(raster_cache()->Draw(mock_layer1.get(), other_canvas));
+  EXPECT_FALSE(raster_cache()->Draw(mock_layer1.get(), cache_canvas));
+  EXPECT_FALSE(raster_cache()->Draw(mock_layer2.get(), other_canvas));
+  EXPECT_FALSE(raster_cache()->Draw(mock_layer2.get(), cache_canvas));
+}
+
 }  // namespace testing
 }  // namespace flutter

flow/layers/opacity_layer.cc

@@ -10,20 +10,7 @@
 namespace flutter {
 
 OpacityLayer::OpacityLayer(SkAlpha alpha, const SkPoint& offset)
-    : alpha_(alpha), offset_(offset) {
-  // Ensure OpacityLayer has only one direct child.
-  //
-  // This is needed to ensure that retained rendering can always be applied to
-  // save the costly saveLayer.
-  //
-  // Any children will be actually added as children of this empty
-  // ContainerLayer.
-  ContainerLayer::Add(std::make_shared<ContainerLayer>());
-}
-
-void OpacityLayer::Add(std::shared_ptr<Layer> layer) {
-  GetChildContainer()->Add(std::move(layer));
-}
+    : alpha_(alpha), offset_(offset) {}
 
 void OpacityLayer::Preroll(PrerollContext* context, const SkMatrix& matrix) {
   TRACE_EVENT0("flutter", "OpacityLayer::Preroll");
@@ -112,19 +99,4 @@ void OpacityLayer::UpdateScene(SceneUpdateContext& context) {
 
 #endif  // defined(OS_FUCHSIA)
 
-ContainerLayer* OpacityLayer::GetChildContainer() const {
-  FML_DCHECK(layers().size() == 1);
-
-  return static_cast<ContainerLayer*>(layers()[0].get());
-}
-
-Layer* OpacityLayer::GetCacheableChild() const {
-  ContainerLayer* child_container = GetChildContainer();
-  if (child_container->layers().size() == 1) {
-    return child_container->layers()[0].get();
-  }
-
-  return child_container;
-}
-
 }  // namespace flutter

flow/layers/opacity_layer.h

@@ -13,7 +13,7 @@ namespace flutter {
 // OpacityLayer is very costly due to the saveLayer call. If there's no child,
 // having the OpacityLayer or not has the same effect. In debug_unopt build,
 // |Preroll| will assert if there are no children.
-class OpacityLayer : public ContainerLayer {
+class OpacityLayer : public MergedContainerLayer {
  public:
   // An offset is provided here because OpacityLayer.addToScene method in the
   // Flutter framework can take an optional offset argument.
@@ -27,8 +27,6 @@ class OpacityLayer : public ContainerLayer {
   // the propagation as repainting the OpacityLayer is expensive.
   OpacityLayer(SkAlpha alpha, const SkPoint& offset);
 
-  void Add(std::shared_ptr<Layer> layer) override;
-
   void Preroll(PrerollContext* context, const SkMatrix& matrix) override;
 
   void Paint(PaintContext& context) const override;
@@ -38,58 +36,6 @@ class OpacityLayer : public ContainerLayer {
 #endif  // defined(OS_FUCHSIA)
 
  private:
-  /**
-   * @brief Returns the ContainerLayer used to hold all of the children
-   * of the OpacityLayer.
-   *
-   * Often opacity layers will only have a single child since the associated
-   * Flutter widget is specified with only a single child widget pointer.
-   * But depending on the structure of the child tree that single widget at
-   * the framework level can turn into multiple children at the engine
-   * API level since there is no guarantee of a 1:1 correspondence of widgets
-   * to engine layers. This synthetic child container layer is established to
-   * hold all of the children in a single layer so that we can cache their
-   * output, but this synthetic layer will typically not be the best choice
-   * for the layer cache since the synthetic container is created fresh with
-   * each new OpacityLayer, and so may not be stable from frame to frame.
-   *
-   * @see GetCacheableChild()
-   * @return the ContainerLayer child used to hold the children
-   */
-  ContainerLayer* GetChildContainer() const;
-
-  /**
-   * @brief Returns the best choice for a Layer object that can be used
-   * in RasterCache operations to cache the children of the OpacityLayer.
-   *
-   * The returned Layer must represent all children and try to remain stable
-   * if the OpacityLayer is reconstructed in subsequent frames of the scene.
-   *
-   * Note that since the synthetic child container returned from the
-   * GetChildContainer() method is created fresh with each new OpacityLayer,
-   * its return value will not be a good candidate for caching. But if the
-   * standard recommendations for animations are followed and the child widget
-   * is wrapped with a RepaintBoundary widget at the framework level, then
-   * the synthetic child container should contain the same single child layer
-   * on each frame. Under those conditions, that single child of the child
-   * container will be the best candidate for caching in the RasterCache
-   * and this method will return that single child if possible to improve
-   * the performance of caching the children.
-   *
-   * Note that if GetCacheableChild() does not find a single stable child of
-   * the child container it will return the child container as a fallback.
-   * Even though that child is new in each frame of an animation and thus we
-   * cannot reuse the cached layer raster between animation frames, the single
-   * container child will allow us to paint the child onto an offscreen buffer
-   * during Preroll() which reduces one render target switch compared to
-   * painting the child on the fly via an AutoSaveLayer in Paint() and thus
-   * still improves our performance.
-   *
-   * @see GetChildContainer()
-   * @return the best candidate Layer for caching the children
-   */
-  Layer* GetCacheableChild() const;
-
   SkAlpha alpha_;
   SkPoint offset_;
   SkRRect frameRRect_;