Add byte streaming capability for the camera (#965)

Author: bparrishMines

packages/camera/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.2.8
+
+* Add access to the image stream from Dart.
+* Use `cameraController.startImageStream(listener)` to process the images.
+
 ## 0.2.7
 
 * Fix issue with crash when the physical device's orientation is unknown.

packages/camera/README.md

@@ -8,6 +8,8 @@ A Flutter plugin for iOS and Android allowing access to the device cameras.
 
 * Display live camera preview in a widget.
 * Snapshots can be captured and saved to a file.
+* Record video.
+* Add access to the image stream from Dart.
 
 ## Installation
 

packages/camera/android/src/main/java/io/flutter/plugins/camera/CameraPlugin.java

@@ -213,6 +213,26 @@ public void onMethodCall(MethodCall call, final Result result) {
           camera.stopVideoRecording(result);
           break;
         }
+      case "startImageStream":
+        {
+          try {
+            camera.startPreviewWithImageStream();
+            result.success(null);
+          } catch (CameraAccessException e) {
+            result.error("CameraAccess", e.getMessage(), null);
+          }
+          break;
+        }
+      case "stopImageStream":
+        {
+          try {
+            camera.startPreview();
+            result.success(null);
+          } catch (CameraAccessException e) {
+            result.error("CameraAccess", e.getMessage(), null);
+          }
+          break;
+        }
       case "dispose":
         {
           if (camera != null) {
@@ -258,7 +278,8 @@ private class Camera {
     private CameraDevice cameraDevice;
     private CameraCaptureSession cameraCaptureSession;
     private EventChannel.EventSink eventSink;
-    private ImageReader imageReader;
+    private ImageReader pictureImageReader;
+    private ImageReader imageStreamReader;
     private int sensorOrientation;
     private boolean isFrontFacing;
     private String cameraName;
@@ -458,9 +479,15 @@ private void open(@Nullable final Result result) {
         if (result != null) result.error("cameraPermission", "Camera permission not granted", null);
       } else {
         try {
-          imageReader =
+          pictureImageReader =
               ImageReader.newInstance(
                   captureSize.getWidth(), captureSize.getHeight(), ImageFormat.JPEG, 2);
+
+          // Used to steam image byte data to dart side.
+          imageStreamReader =
+              ImageReader.newInstance(
+                  previewSize.getWidth(), previewSize.getHeight(), ImageFormat.YUV_420_888, 2);
+
           cameraManager.openCamera(
               cameraName,
               new CameraDevice.StateCallback() {
@@ -553,7 +580,7 @@ private void takePicture(String filePath, @NonNull final Result result) {
         return;
       }
 
-      imageReader.setOnImageAvailableListener(
+      pictureImageReader.setOnImageAvailableListener(
           new ImageReader.OnImageAvailableListener() {
             @Override
             public void onImageAvailable(ImageReader reader) {
@@ -571,7 +598,7 @@ public void onImageAvailable(ImageReader reader) {
       try {
         final CaptureRequest.Builder captureBuilder =
             cameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_STILL_CAPTURE);
-        captureBuilder.addTarget(imageReader.getSurface());
+        captureBuilder.addTarget(pictureImageReader.getSurface());
         captureBuilder.set(CaptureRequest.JPEG_ORIENTATION, getMediaOrientation());
 
         cameraCaptureSession.capture(
@@ -697,7 +724,7 @@ private void startPreview() throws CameraAccessException {
       surfaces.add(previewSurface);
       captureRequestBuilder.addTarget(previewSurface);
 
-      surfaces.add(imageReader.getSurface());
+      surfaces.add(pictureImageReader.getSurface());
 
       cameraDevice.createCaptureSession(
           surfaces,
@@ -727,6 +754,107 @@ public void onConfigureFailed(@NonNull CameraCaptureSession cameraCaptureSession
           null);
     }
 
+    private void startPreviewWithImageStream() throws CameraAccessException {
+      closeCaptureSession();
+
+      SurfaceTexture surfaceTexture = textureEntry.surfaceTexture();
+      surfaceTexture.setDefaultBufferSize(previewSize.getWidth(), previewSize.getHeight());
+
+      captureRequestBuilder =
+          cameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_STILL_CAPTURE);
+
+      List<Surface> surfaces = new ArrayList<>();
+
+      Surface previewSurface = new Surface(surfaceTexture);
+      surfaces.add(previewSurface);
+      captureRequestBuilder.addTarget(previewSurface);
+
+      surfaces.add(imageStreamReader.getSurface());
+      captureRequestBuilder.addTarget(imageStreamReader.getSurface());
+
+      cameraDevice.createCaptureSession(
+          surfaces,
+          new CameraCaptureSession.StateCallback() {
+            @Override
+            public void onConfigured(@NonNull CameraCaptureSession session) {
+              if (cameraDevice == null) {
+                sendErrorEvent("The camera was closed during configuration.");
+                return;
+              }
+              try {
+                cameraCaptureSession = session;
+                captureRequestBuilder.set(
+                    CaptureRequest.CONTROL_MODE, CameraMetadata.CONTROL_MODE_AUTO);
+                cameraCaptureSession.setRepeatingRequest(captureRequestBuilder.build(), null, null);
+              } catch (CameraAccessException e) {
+                sendErrorEvent(e.getMessage());
+              }
+            }
+
+            @Override
+            public void onConfigureFailed(@NonNull CameraCaptureSession cameraCaptureSession) {
+              sendErrorEvent("Failed to configure the camera for streaming images.");
+            }
+          },
+          null);
+
+      registerImageStreamEventChannel();
+    }
+
+    private void registerImageStreamEventChannel() {
+      final EventChannel imageStreamChannel =
+          new EventChannel(registrar.messenger(), "plugins.flutter.io/camera/imageStream");
+
+      imageStreamChannel.setStreamHandler(
+          new EventChannel.StreamHandler() {
+            @Override
+            public void onListen(Object o, EventChannel.EventSink eventSink) {
+              setImageStreamImageAvailableListener(eventSink);
+            }
+
+            @Override
+            public void onCancel(Object o) {
+              imageStreamReader.setOnImageAvailableListener(null, null);
+            }
+          });
+    }
+
+    private void setImageStreamImageAvailableListener(final EventChannel.EventSink eventSink) {
+      imageStreamReader.setOnImageAvailableListener(
+          new ImageReader.OnImageAvailableListener() {
+            @Override
+            public void onImageAvailable(final ImageReader reader) {
+              Image img = reader.acquireLatestImage();
+              if (img == null) return;
+
+              List<Map<String, Object>> planes = new ArrayList<>();
+              for (Image.Plane plane : img.getPlanes()) {
+                ByteBuffer buffer = plane.getBuffer();
+
+                byte[] bytes = new byte[buffer.remaining()];
+                buffer.get(bytes, 0, bytes.length);
+
+                Map<String, Object> planeBuffer = new HashMap<>();
+                planeBuffer.put("bytesPerRow", plane.getRowStride());
+                planeBuffer.put("bytesPerPixel", plane.getPixelStride());
+                planeBuffer.put("bytes", bytes);
+
+                planes.add(planeBuffer);
+              }
+
+              Map<String, Object> imageBuffer = new HashMap<>();
+              imageBuffer.put("width", img.getWidth());
+              imageBuffer.put("height", img.getHeight());
+              imageBuffer.put("format", img.getFormat());
+              imageBuffer.put("planes", planes);
+
+              eventSink.success(imageBuffer);
+              img.close();
+            }
+          },
+          null);
+    }
+
     private void sendErrorEvent(String errorDescription) {
       if (eventSink != null) {
         Map<String, String> event = new HashMap<>();
@@ -750,9 +878,13 @@ private void close() {
         cameraDevice.close();
         cameraDevice = null;
       }
-      if (imageReader != null) {
-        imageReader.close();
-        imageReader = null;
+      if (pictureImageReader != null) {
+        pictureImageReader.close();
+        pictureImageReader = null;
+      }
+      if (imageStreamReader != null) {
+        imageStreamReader.close();
+        imageStreamReader = null;
       }
       if (mediaRecorder != null) {
         mediaRecorder.reset();