diff --git a/packages/firebase_ml_vision/CHANGELOG.md b/packages/firebase_ml_vision/CHANGELOG.md index f898f817ccd6..6632c33013d0 100644 --- a/packages/firebase_ml_vision/CHANGELOG.md +++ b/packages/firebase_ml_vision/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.2.1 + +* Add capability to create image from bytes. + ## 0.2.0+2 * Fix bug with empty text object. diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java index 45cb19fea3c2..3ca61ee4890a 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java @@ -2,6 +2,7 @@ import android.net.Uri; import com.google.firebase.ml.vision.common.FirebaseVisionImage; +import com.google.firebase.ml.vision.common.FirebaseVisionImageMetadata; import io.flutter.plugin.common.MethodCall; import io.flutter.plugin.common.MethodChannel; import io.flutter.plugin.common.MethodChannel.MethodCallHandler; @@ -29,55 +30,74 @@ public static void registerWith(Registrar registrar) { @Override public void onMethodCall(MethodCall call, Result result) { Map options = call.argument("options"); + FirebaseVisionImage image; + Map imageData = call.arguments(); + try { + image = dataToVisionImage(imageData); + } catch (IOException exception) { + result.error("MLVisionDetectorIOError", exception.getLocalizedMessage(), null); + return; + } + switch (call.method) { case "BarcodeDetector#detectInImage": - try { - image = filePathToVisionImage((String) call.argument("path")); - BarcodeDetector.instance.handleDetection(image, options, result); - } catch (IOException e) { - result.error("barcodeDetectorIOError", e.getLocalizedMessage(), null); - } + BarcodeDetector.instance.handleDetection(image, options, result); break; case "FaceDetector#detectInImage": - try { - image = filePathToVisionImage((String) call.argument("path")); - FaceDetector.instance.handleDetection(image, options, result); - } catch (IOException e) { - result.error("faceDetectorIOError", e.getLocalizedMessage(), null); - } + FaceDetector.instance.handleDetection(image, options, result); break; case "LabelDetector#detectInImage": - try { - image = filePathToVisionImage((String) call.argument("path")); - LabelDetector.instance.handleDetection(image, options, result); - } catch (IOException e) { - result.error("labelDetectorIOError", e.getLocalizedMessage(), null); - } + LabelDetector.instance.handleDetection(image, options, result); break; case "CloudLabelDetector#detectInImage": - try { - image = filePathToVisionImage((String) call.argument("path")); - CloudLabelDetector.instance.handleDetection(image, options, result); - } catch (IOException e) { - result.error("cloudLabelDetectorIOError", e.getLocalizedMessage(), null); - } + CloudLabelDetector.instance.handleDetection(image, options, result); break; case "TextRecognizer#processImage": - try { - image = filePathToVisionImage((String) call.argument("path")); - TextRecognizer.instance.handleDetection(image, options, result); - } catch (IOException e) { - result.error("textRecognizerIOError", e.getLocalizedMessage(), null); - } + TextRecognizer.instance.handleDetection(image, options, result); break; default: result.notImplemented(); } } - private FirebaseVisionImage filePathToVisionImage(String path) throws IOException { - File file = new File(path); - return FirebaseVisionImage.fromFilePath(registrar.context(), Uri.fromFile(file)); + private FirebaseVisionImage dataToVisionImage(Map imageData) throws IOException { + String imageType = (String) imageData.get("type"); + + switch (imageType) { + case "file": + File file = new File((String) imageData.get("path")); + return FirebaseVisionImage.fromFilePath(registrar.context(), Uri.fromFile(file)); + case "bytes": + @SuppressWarnings("unchecked") + Map metadataData = (Map) imageData.get("metadata"); + + FirebaseVisionImageMetadata metadata = + new FirebaseVisionImageMetadata.Builder() + .setWidth((int) (double) metadataData.get("width")) + .setHeight((int) (double) metadataData.get("height")) + .setFormat(FirebaseVisionImageMetadata.IMAGE_FORMAT_NV21) + .setRotation(getRotation((int) metadataData.get("rotation"))) + .build(); + + return FirebaseVisionImage.fromByteArray((byte[]) imageData.get("bytes"), metadata); + default: + throw new IllegalArgumentException(String.format("No image type for: %s", imageType)); + } + } + + private int getRotation(int rotation) { + switch (rotation) { + case 0: + return FirebaseVisionImageMetadata.ROTATION_0; + case 90: + return FirebaseVisionImageMetadata.ROTATION_90; + case 180: + return FirebaseVisionImageMetadata.ROTATION_180; + case 270: + return FirebaseVisionImageMetadata.ROTATION_270; + default: + throw new IllegalArgumentException(String.format("No rotation for: %d", rotation)); + } } } diff --git a/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m b/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m index 2b51f3d6a0ff..dcdbf95a9f92 100644 --- a/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m +++ b/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m @@ -36,7 +36,7 @@ - (instancetype)init { } - (void)handleMethodCall:(FlutterMethodCall *)call result:(FlutterResult)result { - FIRVisionImage *image = [self filePathToVisionImage:call.arguments[@"path"]]; + FIRVisionImage *image = [self dataToVisionImage:call.arguments]; NSDictionary *options = call.arguments[@"options"]; if ([@"BarcodeDetector#detectInImage" isEqualToString:call.method]) { [BarcodeDetector handleDetection:image options:options result:result]; @@ -53,8 +53,22 @@ - (void)handleMethodCall:(FlutterMethodCall *)call result:(FlutterResult)result } } -- (FIRVisionImage *)filePathToVisionImage:(NSString *)path { - UIImage *image = [UIImage imageWithContentsOfFile:path]; - return [[FIRVisionImage alloc] initWithImage:image]; +- (FIRVisionImage *)dataToVisionImage:(NSDictionary *)imageData { + NSString *imageType = imageData[@"type"]; + + if ([@"file" isEqualToString:imageType]) { + UIImage *image = [UIImage imageWithContentsOfFile:imageData[@"path"]]; + return [[FIRVisionImage alloc] initWithImage:image]; + } else if ([@"bytes" isEqualToString:imageType]) { + FlutterStandardTypedData *byteData = imageData[@"bytes"]; + NSData *imageBytes = byteData.data; + UIImage *image = [[UIImage alloc] initWithData:imageBytes]; + // TODO(bmparr): Rotate image from imageData[@"rotation"]. + return [[FIRVisionImage alloc] initWithImage:image]; + } else { + NSString *errorReason = [NSString stringWithFormat:@"No image type for: %@", imageType]; + @throw + [NSException exceptionWithName:NSInvalidArgumentException reason:errorReason userInfo:nil]; + } } @end diff --git a/packages/firebase_ml_vision/lib/firebase_ml_vision.dart b/packages/firebase_ml_vision/lib/firebase_ml_vision.dart index 8e228bb65229..00f85da5bfe7 100644 --- a/packages/firebase_ml_vision/lib/firebase_ml_vision.dart +++ b/packages/firebase_ml_vision/lib/firebase_ml_vision.dart @@ -7,6 +7,8 @@ library firebase_ml_vision; import 'dart:async'; import 'dart:io'; import 'dart:math'; +import 'dart:typed_data'; +import 'dart:ui'; import 'package:flutter/foundation.dart'; import 'package:flutter/services.dart'; diff --git a/packages/firebase_ml_vision/lib/src/barcode_detector.dart b/packages/firebase_ml_vision/lib/src/barcode_detector.dart index 8ceedbbe475a..d8705aaeaad6 100644 --- a/packages/firebase_ml_vision/lib/src/barcode_detector.dart +++ b/packages/firebase_ml_vision/lib/src/barcode_detector.dart @@ -189,11 +189,10 @@ class BarcodeDetector extends FirebaseVisionDetector { final List reply = await FirebaseVision.channel.invokeMethod( 'BarcodeDetector#detectInImage', { - 'path': visionImage.imageFile.path, 'options': { 'barcodeFormats': options.barcodeFormats.value, }, - }, + }..addAll(visionImage._serialize()), ); final List barcodes = []; diff --git a/packages/firebase_ml_vision/lib/src/cloud_detector_options.dart b/packages/firebase_ml_vision/lib/src/cloud_detector_options.dart index 4a4144ecd014..58c7c8885fe5 100644 --- a/packages/firebase_ml_vision/lib/src/cloud_detector_options.dart +++ b/packages/firebase_ml_vision/lib/src/cloud_detector_options.dart @@ -29,7 +29,7 @@ class CloudDetectorOptions { /// The type of model to use for the detection. final CloudModelType modelType; - Map _toMap() => { + Map _serialize() => { 'maxResults': maxResults, 'modelType': _enumToString(modelType), }; diff --git a/packages/firebase_ml_vision/lib/src/face_detector.dart b/packages/firebase_ml_vision/lib/src/face_detector.dart index 560f2a7448cf..7d87377d840f 100644 --- a/packages/firebase_ml_vision/lib/src/face_detector.dart +++ b/packages/firebase_ml_vision/lib/src/face_detector.dart @@ -44,7 +44,6 @@ class FaceDetector extends FirebaseVisionDetector { final List reply = await FirebaseVision.channel.invokeMethod( 'FaceDetector#detectInImage', { - 'path': visionImage.imageFile.path, 'options': { 'enableClassification': options.enableClassification, 'enableLandmarks': options.enableLandmarks, @@ -52,7 +51,7 @@ class FaceDetector extends FirebaseVisionDetector { 'minFaceSize': options.minFaceSize, 'mode': _enumToString(options.mode), }, - }, + }..addAll(visionImage._serialize()), ); final List faces = []; diff --git a/packages/firebase_ml_vision/lib/src/firebase_vision.dart b/packages/firebase_ml_vision/lib/src/firebase_vision.dart index 31bd400494b4..6dea5530b70a 100644 --- a/packages/firebase_ml_vision/lib/src/firebase_vision.dart +++ b/packages/firebase_ml_vision/lib/src/firebase_vision.dart @@ -4,6 +4,13 @@ part of firebase_ml_vision; +enum _ImageType { file, bytes } + +/// Indicates the image rotation. +/// +/// Rotation is counter-clockwise. +enum ImageRotation { rotation_0, rotation_90, rotation_180, rotation_270 } + /// The Firebase machine learning vision API. /// /// You can get an instance by calling [FirebaseVision.instance] and then get @@ -56,22 +63,83 @@ class FirebaseVision { /// /// Create an instance by calling one of the factory constructors. class FirebaseVisionImage { - FirebaseVisionImage._(this.imageFile); + const FirebaseVisionImage._({ + @required _ImageType type, + FirebaseVisionImageMetadata metadata, + File imageFile, + Uint8List bytes, + }) : _imageFile = imageFile, + _metadata = metadata, + _bytes = bytes, + _type = type; + // TODO(bmparr): Add [ImageOrientation] when passing file. /// Construct a [FirebaseVisionImage] from a file. factory FirebaseVisionImage.fromFile(File imageFile) { assert(imageFile != null); - return FirebaseVisionImage._(imageFile); + return FirebaseVisionImage._( + type: _ImageType.file, + imageFile: imageFile, + ); } /// Construct a [FirebaseVisionImage] from a file path. factory FirebaseVisionImage.fromFilePath(String imagePath) { assert(imagePath != null); - return FirebaseVisionImage._(File(imagePath)); + return FirebaseVisionImage._( + type: _ImageType.file, + imageFile: File(imagePath), + ); + } + + /// Construct a [FirebaseVisionImage] from a list of bytes. + /// + /// Expects `ImageFormat.NV21` on Android and expects bytes from `NSData` + /// provided from a `UIImage` on iOS. (e.g. using + /// `UIImageJPEGRepresentation()` on the platform side). + factory FirebaseVisionImage.fromBytes( + Uint8List bytes, + FirebaseVisionImageMetadata metadata, + ) { + assert(bytes != null); + return FirebaseVisionImage._( + type: _ImageType.bytes, + bytes: bytes, + metadata: metadata, + ); } - /// The file location of the image. - final File imageFile; + final Uint8List _bytes; + final File _imageFile; + final FirebaseVisionImageMetadata _metadata; + final _ImageType _type; + + Map _serialize() => { + 'type': _enumToString(_type), + 'bytes': _bytes, + 'path': _imageFile?.path, + 'metadata': _type == _ImageType.bytes ? _metadata._serialize() : null, + }; +} + +/// Image metadata used by [FirebaseVision] detectors. +/// +/// [rotation] defaults to [ImageRotation.rotation_0]. Currently only rotates on +/// Android. +class FirebaseVisionImageMetadata { + const FirebaseVisionImageMetadata({ + @required this.size, + this.rotation = ImageRotation.rotation_0, + }) : assert(size != null); + + final Size size; + final ImageRotation rotation; + + Map _serialize() => { + 'width': size.width, + 'height': size.height, + 'rotation': _imageRotationToInt(rotation), + }; } /// Abstract class for detectors in [FirebaseVision] API. @@ -80,6 +148,20 @@ abstract class FirebaseVisionDetector { Future detectInImage(FirebaseVisionImage visionImage); } +int _imageRotationToInt(ImageRotation rotation) { + switch (rotation) { + case ImageRotation.rotation_90: + return 90; + case ImageRotation.rotation_180: + return 180; + case ImageRotation.rotation_270: + return 270; + default: + assert(rotation == ImageRotation.rotation_0); + return 0; + } +} + String _enumToString(dynamic enumValue) { final String enumString = enumValue.toString(); return enumString.substring(enumString.indexOf('.') + 1); diff --git a/packages/firebase_ml_vision/lib/src/label_detector.dart b/packages/firebase_ml_vision/lib/src/label_detector.dart index 8511724406d0..9adaaae80b43 100644 --- a/packages/firebase_ml_vision/lib/src/label_detector.dart +++ b/packages/firebase_ml_vision/lib/src/label_detector.dart @@ -34,11 +34,10 @@ class LabelDetector extends FirebaseVisionDetector { final List reply = await FirebaseVision.channel.invokeMethod( 'LabelDetector#detectInImage', { - 'path': visionImage.imageFile.path, 'options': { 'confidenceThreshold': options.confidenceThreshold, }, - }, + }..addAll(visionImage._serialize()), ); final List