diff --git a/packages/camera/android/src/main/java/io/flutter/plugins/camera/CameraPlugin.java b/packages/camera/android/src/main/java/io/flutter/plugins/camera/CameraPlugin.java index 61ea4fff3560..77f0068db92f 100644 --- a/packages/camera/android/src/main/java/io/flutter/plugins/camera/CameraPlugin.java +++ b/packages/camera/android/src/main/java/io/flutter/plugins/camera/CameraPlugin.java @@ -203,6 +203,26 @@ public void onMethodCall(MethodCall call, final Result result) { camera.stopVideoRecording(result); break; } + case "startByteStream": + { + try { + camera.startPreviewWithByteStream(); + result.success(null); + } catch (CameraAccessException e) { + result.error("CameraAccess", e.getMessage(), null); + } + break; + } + case "stopByteStream": + { + try { + camera.startPreview(); + result.success(null); + } catch (CameraAccessException e) { + result.error("CameraAccess", e.getMessage(), null); + } + break; + } case "dispose": { if (camera != null) { @@ -248,7 +268,8 @@ private class Camera { private CameraDevice cameraDevice; private CameraCaptureSession cameraCaptureSession; private EventChannel.EventSink eventSink; - private ImageReader imageReader; + private ImageReader pictureImageReader; + private ImageReader byteImageReader; // Used to pass bytes to dart side. private int sensorOrientation; private boolean isFrontFacing; private String cameraName; @@ -453,9 +474,13 @@ private void open(@Nullable final Result result) { if (result != null) result.error("cameraPermission", "Camera permission not granted", null); } else { try { - imageReader = + pictureImageReader = ImageReader.newInstance( captureSize.getWidth(), captureSize.getHeight(), ImageFormat.JPEG, 2); + byteImageReader = + ImageReader.newInstance( + previewSize.getWidth(), previewSize.getHeight(), ImageFormat.YUV_420_888, 2); + cameraManager.openCamera( cameraName, new CameraDevice.StateCallback() { @@ -548,7 +573,7 @@ private void takePicture(String filePath, @NonNull final Result result) { return; } - imageReader.setOnImageAvailableListener( + pictureImageReader.setOnImageAvailableListener( new ImageReader.OnImageAvailableListener() { @Override public void onImageAvailable(ImageReader reader) { @@ -566,7 +591,7 @@ public void onImageAvailable(ImageReader reader) { try { final CaptureRequest.Builder captureBuilder = cameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_STILL_CAPTURE); - captureBuilder.addTarget(imageReader.getSurface()); + captureBuilder.addTarget(pictureImageReader.getSurface()); int displayRotation = activity.getWindowManager().getDefaultDisplay().getRotation(); int displayOrientation = ORIENTATIONS.get(displayRotation); if (isFrontFacing) displayOrientation = -displayOrientation; @@ -696,7 +721,7 @@ private void startPreview() throws CameraAccessException { surfaces.add(previewSurface); captureRequestBuilder.addTarget(previewSurface); - surfaces.add(imageReader.getSurface()); + surfaces.add(pictureImageReader.getSurface()); cameraDevice.createCaptureSession( surfaces, @@ -726,6 +751,107 @@ public void onConfigureFailed(@NonNull CameraCaptureSession cameraCaptureSession null); } + private void startPreviewWithByteStream() throws CameraAccessException { + closeCaptureSession(); + + SurfaceTexture surfaceTexture = textureEntry.surfaceTexture(); + surfaceTexture.setDefaultBufferSize(previewSize.getWidth(), previewSize.getHeight()); + + captureRequestBuilder = + cameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_STILL_CAPTURE); + + List surfaces = new ArrayList<>(); + + Surface previewSurface = new Surface(surfaceTexture); + surfaces.add(previewSurface); + captureRequestBuilder.addTarget(previewSurface); + + surfaces.add(byteImageReader.getSurface()); + captureRequestBuilder.addTarget(byteImageReader.getSurface()); + + cameraDevice.createCaptureSession( + surfaces, + new CameraCaptureSession.StateCallback() { + @Override + public void onConfigured(@NonNull CameraCaptureSession session) { + if (cameraDevice == null) { + sendErrorEvent("The camera was closed during configuration."); + return; + } + try { + cameraCaptureSession = session; + captureRequestBuilder.set( + CaptureRequest.CONTROL_MODE, CameraMetadata.CONTROL_MODE_AUTO); + cameraCaptureSession.setRepeatingRequest(captureRequestBuilder.build(), null, null); + } catch (CameraAccessException e) { + sendErrorEvent(e.getMessage()); + } + } + + @Override + public void onConfigureFailed(@NonNull CameraCaptureSession cameraCaptureSession) { + sendErrorEvent("Failed to configure the camera for streaming bytes."); + } + }, + null); + + registerByteStreamEventChannel(); + } + + private void registerByteStreamEventChannel() { + final EventChannel cameraChannel = + new EventChannel(registrar.messenger(), "plugins.flutter.io/camera/bytes"); + + cameraChannel.setStreamHandler( + new EventChannel.StreamHandler() { + @Override + public void onListen(Object o, EventChannel.EventSink eventSink) { + setByteStreamImageAvailableListener(eventSink); + } + + @Override + public void onCancel(Object o) { + byteImageReader.setOnImageAvailableListener(null, null); + } + }); + } + + private void setByteStreamImageAvailableListener(final EventChannel.EventSink eventSink) { + byteImageReader.setOnImageAvailableListener( + new ImageReader.OnImageAvailableListener() { + @Override + public void onImageAvailable(final ImageReader reader) { + Image img = reader.acquireLatestImage(); + if (img == null) return; + + eventSink.success(YUV_420_888toNV21(img)); + img.close(); + } + }, + null); + } + + private byte[] YUV_420_888toNV21(Image image) { + byte[] nv21; + + ByteBuffer yBuffer = image.getPlanes()[0].getBuffer(); + ByteBuffer uBuffer = image.getPlanes()[1].getBuffer(); + ByteBuffer vBuffer = image.getPlanes()[2].getBuffer(); + + int ySize = yBuffer.remaining(); + int uSize = uBuffer.remaining(); + int vSize = vBuffer.remaining(); + + nv21 = new byte[ySize + uSize + vSize]; + + //U and V are swapped + yBuffer.get(nv21, 0, ySize); + vBuffer.get(nv21, ySize, vSize); + uBuffer.get(nv21, ySize + vSize, uSize); + + return nv21; + } + private void sendErrorEvent(String errorDescription) { if (eventSink != null) { Map event = new HashMap<>(); @@ -749,9 +875,13 @@ private void close() { cameraDevice.close(); cameraDevice = null; } - if (imageReader != null) { - imageReader.close(); - imageReader = null; + if (pictureImageReader != null) { + pictureImageReader.close(); + pictureImageReader = null; + } + if (byteImageReader != null) { + byteImageReader.close(); + byteImageReader = null; } if (mediaRecorder != null) { mediaRecorder.reset(); diff --git a/packages/camera/example/ios/Runner.xcodeproj/project.pbxproj b/packages/camera/example/ios/Runner.xcodeproj/project.pbxproj index 5a54057fee45..f3ac434ae2e7 100644 --- a/packages/camera/example/ios/Runner.xcodeproj/project.pbxproj +++ b/packages/camera/example/ios/Runner.xcodeproj/project.pbxproj @@ -161,7 +161,6 @@ 9705A1C41CF9048500538489 /* Embed Frameworks */, 3B06AD1E1E4923F5004D2608 /* Thin Binary */, FE224661708E6DA2A0F8B952 /* [CP] Embed Pods Frameworks */, - EACF0929FF12B6CC70C2D6BE /* [CP] Copy Pods Resources */, ); buildRules = ( ); @@ -183,7 +182,7 @@ TargetAttributes = { 97C146ED1CF9000F007C117D = { CreatedOnToolsVersion = 7.3.1; - DevelopmentTeam = EQHXZ8M8AV; + DevelopmentTeam = S8QB4VV633; }; }; }; @@ -269,21 +268,6 @@ shellPath = /bin/sh; shellScript = "/bin/sh \"$FLUTTER_ROOT/packages/flutter_tools/bin/xcode_backend.sh\" build"; }; - EACF0929FF12B6CC70C2D6BE /* [CP] Copy Pods Resources */ = { - isa = PBXShellScriptBuildPhase; - buildActionMask = 2147483647; - files = ( - ); - inputPaths = ( - ); - name = "[CP] Copy Pods Resources"; - outputPaths = ( - ); - runOnlyForDeploymentPostprocessing = 0; - shellPath = /bin/sh; - shellScript = "\"${SRCROOT}/Pods/Target Support Files/Pods-Runner/Pods-Runner-resources.sh\"\n"; - showEnvVarsInLog = 0; - }; FE224661708E6DA2A0F8B952 /* [CP] Embed Pods Frameworks */ = { isa = PBXShellScriptBuildPhase; buildActionMask = 2147483647; @@ -291,7 +275,7 @@ ); inputPaths = ( "${SRCROOT}/Pods/Target Support Files/Pods-Runner/Pods-Runner-frameworks.sh", - "${PODS_ROOT}/../../../../../../flutter/bin/cache/artifacts/engine/ios-release/Flutter.framework", + "${PODS_ROOT}/../.symlinks/flutter/ios/Flutter.framework", ); name = "[CP] Embed Pods Frameworks"; outputPaths = ( @@ -433,7 +417,7 @@ buildSettings = { ARCHS = arm64; ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; - DEVELOPMENT_TEAM = EQHXZ8M8AV; + DEVELOPMENT_TEAM = S8QB4VV633; ENABLE_BITCODE = NO; FRAMEWORK_SEARCH_PATHS = ( "$(inherited)", @@ -456,7 +440,7 @@ buildSettings = { ARCHS = arm64; ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; - DEVELOPMENT_TEAM = EQHXZ8M8AV; + DEVELOPMENT_TEAM = S8QB4VV633; ENABLE_BITCODE = NO; FRAMEWORK_SEARCH_PATHS = ( "$(inherited)", diff --git a/packages/camera/ios/Classes/CameraPlugin.m b/packages/camera/ios/Classes/CameraPlugin.m index 0bfb7515c36c..c17a25ef9ae1 100644 --- a/packages/camera/ios/Classes/CameraPlugin.m +++ b/packages/camera/ios/Classes/CameraPlugin.m @@ -21,6 +21,23 @@ @interface FLTSavePhotoDelegate : NSObject - initWithPath:(NSString *)filename result:(FlutterResult)result; @end +@interface FLTByteStreamHandler : NSObject +@property(readonly, nonatomic) FlutterEventSink eventSink; +@end + +@implementation FLTByteStreamHandler { +} +- (FlutterError *_Nullable)onCancelWithArguments:(id _Nullable)arguments { + return nil; +} + +- (FlutterError *_Nullable)onListenWithArguments:(id _Nullable)arguments + eventSink:(nonnull FlutterEventSink)events { + _eventSink = events; + return nil; +} +@end + @implementation FLTSavePhotoDelegate { /// Used to keep the delegate alive until didFinishProcessingPhotoSampleBuffer. FLTSavePhotoDelegate *selfReference; @@ -64,6 +81,7 @@ @interface FLTCam : NSObject *)messenger { + if (!_isStreamingBytes) { + FlutterEventChannel *eventChannel = + [FlutterEventChannel eventChannelWithName:@"plugins.flutter.io/camera/bytes" + binaryMessenger:messenger]; + + _byteStreamHandler = [[FLTByteStreamHandler alloc] init]; + [eventChannel setStreamHandler:_byteStreamHandler]; + + _isStreamingBytes = YES; + } else { + _eventSink( + @{@"event" : @"error", @"errorDescription" : @"Bytes from camera are already streaming!"}); + } +} + +- (void)stopByteStream { + if (_isStreamingBytes) { + _isStreamingBytes = NO; + _byteStreamHandler = nil; + } else { + _eventSink( + @{@"event" : @"error", @"errorDescription" : @"Bytes from camera are not streaming!"}); + } +} + - (BOOL)setupWriterForPath:(NSString *)path { NSError *error = nil; NSURL *outputURL; @@ -492,6 +581,12 @@ - (void)handleMethodCall:(FlutterMethodCall *)call result:(FlutterResult)result }); [cam start]; } + } else if ([@"startByteStream" isEqualToString:call.method]) { + [_camera startByteStreamWithMessenger:_messenger]; + result(nil); + } else if ([@"stopByteStream" isEqualToString:call.method]) { + [_camera stopByteStream]; + result(nil); } else { NSDictionary *argsMap = call.arguments; NSUInteger textureId = ((NSNumber *)argsMap[@"textureId"]).unsignedIntegerValue; diff --git a/packages/camera/lib/camera.dart b/packages/camera/lib/camera.dart index 153bbeb69b98..e8c6538e61d5 100644 --- a/packages/camera/lib/camera.dart +++ b/packages/camera/lib/camera.dart @@ -1,4 +1,5 @@ import 'dart:async'; +import 'dart:typed_data'; import 'package:flutter/services.dart'; import 'package:flutter/widgets.dart'; @@ -10,6 +11,8 @@ enum CameraLensDirection { front, back, external } enum ResolutionPreset { low, medium, high } +typedef void OnLatestImageAvailable(Uint8List bytes); + /// Returns the resolution preset as a String. String serializeResolutionPreset(ResolutionPreset resolutionPreset) { switch (resolutionPreset) { @@ -110,13 +113,15 @@ class CameraValue { this.previewSize, this.isRecordingVideo, this.isTakingPicture, + this.isStreamingBytes, }); const CameraValue.uninitialized() : this( isInitialized: false, isRecordingVideo: false, - isTakingPicture: false); + isTakingPicture: false, + isStreamingBytes: false); /// True after [CameraController.initialize] has completed successfully. final bool isInitialized; @@ -127,6 +132,9 @@ class CameraValue { /// True when the camera is recording (not the same as previewing). final bool isRecordingVideo; + /// True when bytes from the camera are being streamed. + final bool isStreamingBytes; + final String errorDescription; /// The size of the preview in pixels. @@ -145,6 +153,7 @@ class CameraValue { bool isInitialized, bool isRecordingVideo, bool isTakingPicture, + bool isStreamingBytes, String errorDescription, Size previewSize, }) { @@ -154,6 +163,7 @@ class CameraValue { previewSize: previewSize ?? this.previewSize, isRecordingVideo: isRecordingVideo ?? this.isRecordingVideo, isTakingPicture: isTakingPicture ?? this.isTakingPicture, + isStreamingBytes: isStreamingBytes ?? this.isStreamingBytes, ); } @@ -164,7 +174,8 @@ class CameraValue { 'isRecordingVideo: $isRecordingVideo, ' 'isInitialized: $isInitialized, ' 'errorDescription: $errorDescription, ' - 'previewSize: $previewSize)'; + 'previewSize: $previewSize, ' + 'isStreamingBytes: $isStreamingBytes)'; } } @@ -185,6 +196,7 @@ class CameraController extends ValueNotifier { int _textureId; bool _isDisposed = false; StreamSubscription _eventSubscription; + StreamSubscription _byteStreamSubscription; Completer _creatingCompleter; /// Initializes the camera on the device. @@ -276,6 +288,71 @@ class CameraController extends ValueNotifier { } } + Future startByteStream(OnLatestImageAvailable onAvailable) async { + if (!value.isInitialized || _isDisposed) { + throw CameraException( + 'Uninitialized CameraController', + 'startByteStream was called on uninitialized CameraController.', + ); + } + if (value.isRecordingVideo) { + throw CameraException( + 'A video recording is already started.', + 'startByteStream was called while a video is being recorded.', + ); + } + if (value.isStreamingBytes) { + throw CameraException( + 'A camera has started streaming bytes.', + 'startByteStream was called while a camera was streaming bytes.', + ); + } + + try { + await _channel.invokeMethod('startByteStream'); + value = value.copyWith(isStreamingBytes: true); + } on PlatformException catch (e) { + throw CameraException(e.code, e.message); + } + const EventChannel cameraEventChannel = + EventChannel('plugins.flutter.io/camera/bytes'); + _byteStreamSubscription = + cameraEventChannel.receiveBroadcastStream().listen((dynamic bytes) { + onAvailable(bytes); + }); + } + + Future stopByteStream() async { + if (!value.isInitialized || _isDisposed) { + throw CameraException( + 'Uninitialized CameraController', + 'stopByteStream was called on uninitialized CameraController.', + ); + } + if (value.isRecordingVideo) { + throw CameraException( + 'A video recording is already started.', + 'stopByteStream was called while a video is being recorded.', + ); + } + if (!value.isStreamingBytes) { + throw CameraException( + 'No camera is streaming bytes', + 'stopByteStream was called when no camera is streaming bytes.', + ); + } + + try { + value = value.copyWith(isStreamingBytes: false); + await _channel.invokeMethod('stopByteStream'); + } on PlatformException catch (e) { + throw CameraException(e.code, e.message); + } + + _byteStreamSubscription.cancel(); + _byteStreamSubscription = null; + } + /// Start a video recording and save the file to [path]. /// /// A path can for example be obtained using @@ -299,6 +376,13 @@ class CameraController extends ValueNotifier { 'startVideoRecording was called when a recording is already started.', ); } + if (value.isStreamingBytes) { + throw CameraException( + 'A camera has started streaming bytes.', + 'startVideoRecording was called while a camera was streaming bytes.', + ); + } + try { await _channel.invokeMethod( 'startVideoRecording',