Workaround HardwareRenderer breakage in Android 14 (flutter#52370)

johnmccutchan · reidbaker · commit 0e812e96b940 · 2024-04-30T11:21:38.000-04:00
- Destroy ImageReaders on memory trim. - Unset the VirtualDisplay's surface on memory trim. - On resume, recreate ImageReaders. - On resume, do a dumb little dance and then set the VirtualDisplay's surface Fixes: flutter/flutter#146499 Fixes: flutter/flutter#144219 Internal bug: b/335646931 Android Fix: https://googleplex-android-review.git.corp.google.com/c/platform/frameworks/base/+/27015418 Android 15 will include the fix. Unclear if Android 14 will be patched.
diff --git a/shell/platform/android/io/flutter/embedding/android/FlutterActivityAndFragmentDelegate.java b/shell/platform/android/io/flutter/embedding/android/FlutterActivityAndFragmentDelegate.java
@@ -578,6 +578,7 @@ void onPostResume() {
     ensureAlive();
     if (flutterEngine != null) {
       updateSystemUiOverlays();
+      flutterEngine.getPlatformViewsController().onResume();
     } else {
       Log.w(TAG, "onPostResume() invoked before FlutterFragment was attached to an Activity.");
     }
@@ -921,6 +922,7 @@ void onTrimMemory(int level) {
         flutterEngine.getSystemChannel().sendMemoryPressureWarning();
       }
       flutterEngine.getRenderer().onTrimMemory(level);
+      flutterEngine.getPlatformViewsController().onTrimMemory(level);
     }
   }
 
diff --git a/shell/platform/android/io/flutter/embedding/engine/renderer/FlutterRenderer.java b/shell/platform/android/io/flutter/embedding/engine/renderer/FlutterRenderer.java
@@ -5,6 +5,7 @@
 package io.flutter.embedding.engine.renderer;
 
 import android.annotation.TargetApi;
+import android.content.ComponentCallbacks2;
 import android.graphics.Bitmap;
 import android.graphics.ImageFormat;
 import android.graphics.Rect;
@@ -361,17 +362,47 @@ public void run() {
   final class ImageReaderSurfaceProducer
       implements TextureRegistry.SurfaceProducer, TextureRegistry.ImageConsumer {
     private static final String TAG = "ImageReaderSurfaceProducer";
-    private static final int MAX_IMAGES = 4;
+    private static final int MAX_IMAGES = 5;
+
+    // Flip when debugging to see verbose logs.
+    private static final boolean VERBOSE_LOGS = false;
+
+    // We must always cleanup on memory pressure on Android 14 due to a bug in Android.
+    // It is safe to do on all versions so we unconditionally have this set to true.
+    private static final boolean CLEANUP_ON_MEMORY_PRESSURE = true;
 
     private final long id;
 
     private boolean released;
     private boolean ignoringFence = false;
 
-    private int requestedWidth = 0;
-    private int requestedHeight = 0;
-
     /** Internal class: state held per image produced by image readers. */
+    private boolean trimOnMemoryPressure = CLEANUP_ON_MEMORY_PRESSURE;
+
+    // The requested width and height are updated by setSize.
+    private int requestedWidth = 1;
+    private int requestedHeight = 1;
+    // Whenever the requested width and height change we set this to be true so we
+    // create a new ImageReader (inside getSurface) with the correct width and height.
+    // We use this flag so that we lazily create the ImageReader only when a frame
+    // will be produced at that size.
+    private boolean createNewReader = true;
+
+    // State held to track latency of various stages.
+    private long lastDequeueTime = 0;
+    private long lastQueueTime = 0;
+    private long lastScheduleTime = 0;
+    private int numTrims = 0;
+
+    private Object lock = new Object();
+    // REQUIRED: The following fields must only be accessed when lock is held.
+    private final ArrayDeque<PerImageReader> imageReaderQueue = new ArrayDeque<PerImageReader>();
+    private final HashMap<ImageReader, PerImageReader> perImageReaders =
+        new HashMap<ImageReader, PerImageReader>();
+    private PerImage lastDequeuedImage = null;
+    private PerImageReader lastReaderDequeuedFrom = null;
+
+    /** Internal class: state held per Image produced by ImageReaders. */
     private class PerImage {
       public final ImageReader reader;
       public final Image image;
@@ -414,9 +445,184 @@ public void onImageAvailable(ImageReader reader) {
               return;
             }
             onImage(new PerImage(reader, image));
-          }
+          };
         };
 
+    PerImage dequeueImage() {
+      if (imageQueue.size() == 0) {
+        return null;
+      }
+      PerImage r = imageQueue.removeFirst();
+      return r;
+    }
+
+    /** returns true if we can prune this reader */
+    boolean canPrune() {
+      return imageQueue.size() == 0 && lastReaderDequeuedFrom != this;
+    }
+
+    void close() {
+      closed = true;
+      if (VERBOSE_LOGS) {
+        Log.i(TAG, "Closing reader=" + reader.hashCode());
+      }
+      reader.close();
+      imageQueue.clear();
+    }
+
+    double deltaMillis(long deltaNanos) {
+      double ms = (double) deltaNanos / (double) 1000000.0;
+      return ms;
+    }
+
+    PerImageReader getOrCreatePerImageReader(ImageReader reader) {
+      PerImageReader r = perImageReaders.get(reader);
+      if (r == null) {
+        r = new PerImageReader(reader);
+        perImageReaders.put(reader, r);
+        imageReaderQueue.add(r);
+        if (VERBOSE_LOGS) {
+          Log.i(TAG, "imageReaderQueue#=" + imageReaderQueue.size());
+        }
+      }
+      return r;
+    }
+
+    void pruneImageReaderQueue() {
+      boolean change = false;
+      // Prune nodes from the head of the ImageReader queue.
+      while (imageReaderQueue.size() > 1) {
+        PerImageReader r = imageReaderQueue.peekFirst();
+        if (!r.canPrune()) {
+          // No more ImageReaders can be pruned this round.
+          break;
+        }
+        imageReaderQueue.removeFirst();
+        perImageReaders.remove(r.reader);
+        r.close();
+        change = true;
+      }
+      if (change && VERBOSE_LOGS) {
+        Log.i(TAG, "Pruned image reader queue length=" + imageReaderQueue.size());
+      }
+    }
+
+    void onImage(ImageReader reader, Image image) {
+      PerImage queuedImage = null;
+      synchronized (lock) {
+        PerImageReader perReader = getOrCreatePerImageReader(reader);
+        queuedImage = perReader.queueImage(image);
+      }
+      if (queuedImage == null) {
+        // We got a late image.
+        return;
+      }
+      if (VERBOSE_LOGS) {
+        if (lastQueueTime != 0) {
+          long now = System.nanoTime();
+          long queueDelta = now - lastQueueTime;
+          Log.i(
+              TAG,
+              ""
+                  + reader.hashCode()
+                  + " enqueued image="
+                  + queuedImage.image.hashCode()
+                  + " queueDelta="
+                  + deltaMillis(queueDelta));
+          lastQueueTime = now;
+        } else {
+          lastQueueTime = System.nanoTime();
+        }
+      }
+      scheduleEngineFrame();
+    }
+
+    public PerImageReader(ImageReader reader) {
+        this.reader = reader;
+        reader.setOnImageAvailableListener(
+            onImageAvailableListener, new Handler(Looper.getMainLooper()));
+    }
+
+    PerImage queueImage(Image image) {
+      if (closed) {
+        return null;
+      }
+      PerImage perImage = new PerImage(image, System.nanoTime());
+      imageQueue.add(perImage);
+      // If we fall too far behind we will skip some frames.
+      while (imageQueue.size() > 2) {
+        PerImage r = imageQueue.removeFirst();
+        if (VERBOSE_LOGS) {
+          Log.i(TAG, "" + reader.hashCode() + " force closed image=" + r.image.hashCode());
+        }
+        r.image.close();
+      }
+      return perImage;
+    }
+
+    @Override
+    public void onTrimMemory(int level) {
+      if (!trimOnMemoryPressure) {
+        return;
+      }
+      if (level < ComponentCallbacks2.TRIM_MEMORY_BACKGROUND) {
+        return;
+      }
+      synchronized (lock) {
+        numTrims++;
+      }
+      cleanup();
+      createNewReader = true;
+    }
+
+    private void cleanup() {
+      synchronized (lock) {
+        for (PerImageReader pir : perImageReaders.values()) {
+          if (lastReaderDequeuedFrom == pir) {
+            lastReaderDequeuedFrom = null;
+          }
+          pir.close();
+        }
+        perImageReaders.clear();
+        if (lastDequeuedImage != null) {
+          lastDequeuedImage.image.close();
+          lastDequeuedImage = null;
+        }
+        if (lastReaderDequeuedFrom != null) {
+          lastReaderDequeuedFrom.close();
+          lastReaderDequeuedFrom = null;
+        }
+        imageReaderQueue.clear();
+      }
+    }
+
+    @TargetApi(API_LEVELS.API_33)
+    private void waitOnFence(Image image) {
+      try {
+        SyncFence fence = image.getFence();
+        fence.awaitForever();
+      } catch (IOException e) {
+        // Drop.
+      }
+    }
+
+    private void maybeWaitOnFence(Image image) {
+      if (image == null) {
+        return;
+      }
+      if (ignoringFence) {
+        return;
+      }
+      if (Build.VERSION.SDK_INT >= API_LEVELS.API_33) {
+        // The fence API is only available on Android >= 33.
+        waitOnFence(image);
+        return;
+      }
+      // Log once per ImageTextureEntry.
+      ignoringFence = true;
+      Log.w(TAG, "ImageTextureEntry can't wait on the fence on Android < 33");
+    }
+
     ImageReaderSurfaceProducer(long id) {
       this.id = id;
     }
@@ -662,8 +868,28 @@ public void disableFenceForTest() {
     }
 
     @VisibleForTesting
-    public int readersToCloseSize() {
-      return readersToClose.size();
+    public int numImageReaders() {
+      synchronized (lock) {
+        return imageReaderQueue.size();
+      }
+    }
+
+    @VisibleForTesting
+    public int numTrims() {
+      synchronized (lock) {
+        return numTrims;
+      }
+    }
+
+    @VisibleForTesting
+    public int numImages() {
+      int r = 0;
+      synchronized (lock) {
+        for (PerImageReader reader : imageReaderQueue) {
+          r += reader.imageQueue.size();
+        }
+      }
+      return r;
     }
   }
 
diff --git a/shell/platform/android/io/flutter/plugin/platform/PlatformViewsController.java b/shell/platform/android/io/flutter/plugin/platform/PlatformViewsController.java
@@ -8,6 +8,7 @@
 import static android.view.MotionEvent.PointerProperties;
 
 import android.annotation.TargetApi;
+import android.content.ComponentCallbacks2;
 import android.content.Context;
 import android.content.MutableContextWrapper;
 import android.os.Build;
@@ -1056,6 +1057,24 @@ private void diposeAllViews() {
     }
   }
 
+  // Invoked when the Android system is requesting we reduce memory usage.
+  public void onTrimMemory(int level) {
+    if (level < ComponentCallbacks2.TRIM_MEMORY_BACKGROUND) {
+      return;
+    }
+    for (VirtualDisplayController vdc : vdControllers.values()) {
+      vdc.clearSurface();
+    }
+  }
+
+  // Called after the application has been resumed.
+  // This is where we undo whatever may have been done in onTrimMemory.
+  public void onResume() {
+    for (VirtualDisplayController vdc : vdControllers.values()) {
+      vdc.resetSurface();
+    }
+  }
+
   /**
    * Disposes a single
    *
diff --git a/shell/platform/android/io/flutter/plugin/platform/VirtualDisplayController.java b/shell/platform/android/io/flutter/plugin/platform/VirtualDisplayController.java
@@ -261,6 +261,49 @@ public void dispatchTouchEvent(MotionEvent event) {
     presentation.dispatchTouchEvent(event);
   }
 
+  public void clearSurface() {
+    virtualDisplay.setSurface(null);
+  }
+
+  public void resetSurface() {
+    final int width = getRenderTargetWidth();
+    final int height = getRenderTargetHeight();
+    final boolean isFocused = getView().isFocused();
+    final SingleViewPresentation.PresentationState presentationState = presentation.detachState();
+
+    // We detach the surface to prevent it being destroyed when releasing the vd.
+    virtualDisplay.setSurface(null);
+    virtualDisplay.release();
+    final DisplayManager displayManager =
+        (DisplayManager) context.getSystemService(Context.DISPLAY_SERVICE);
+    int flags = 0;
+    virtualDisplay =
+        displayManager.createVirtualDisplay(
+            "flutter-vd#" + viewId,
+            width,
+            height,
+            densityDpi,
+            renderTarget.getSurface(),
+            flags,
+            callback,
+            null /* handler */);
+    // Create a new SingleViewPresentation and show() it before we cancel() the existing
+    // presentation. Calling show() and cancel() in this order fixes
+    // https://github.com/flutter/flutter/issues/26345 and maintains seamless transition
+    // of the contents of the presentation.
+    SingleViewPresentation newPresentation =
+        new SingleViewPresentation(
+            context,
+            virtualDisplay.getDisplay(),
+            accessibilityEventsDelegate,
+            presentationState,
+            focusChangeListener,
+            isFocused);
+    newPresentation.show();
+    presentation.cancel();
+    presentation = newPresentation;
+  }
+
   static class OneTimeOnDrawListener implements ViewTreeObserver.OnDrawListener {
     static void schedule(View view, Runnable runnable) {
       OneTimeOnDrawListener listener = new OneTimeOnDrawListener(view, runnable);
diff --git a/shell/platform/android/test/io/flutter/embedding/engine/renderer/FlutterRendererTest.java b/shell/platform/android/test/io/flutter/embedding/engine/renderer/FlutterRendererTest.java

Original file line number	Diff line number	Diff line change
`@@ -578,6 +578,7 @@ void onPostResume() {`
`578`	`578`	`ensureAlive();`
`579`	`579`	`if (flutterEngine != null) {`
`580`	`580`	`updateSystemUiOverlays();`
	`581`	`+ flutterEngine.getPlatformViewsController().onResume();`
`581`	`582`	`} else {`
`582`	`583`	`Log.w(TAG, "onPostResume() invoked before FlutterFragment was attached to an Activity.");`
`583`	`584`	`}`
`@@ -921,6 +922,7 @@ void onTrimMemory(int level) {`
`921`	`922`	`flutterEngine.getSystemChannel().sendMemoryPressureWarning();`
`922`	`923`	`}`
`923`	`924`	`flutterEngine.getRenderer().onTrimMemory(level);`
	`925`	`+ flutterEngine.getPlatformViewsController().onTrimMemory(level);`
`924`	`926`	`}`
`925`	`927`	`}`
`926`	`928`