Add a mechanism for testing test results

Author: jiahaog

packages/e2e/lib/e2e.dart

@@ -32,7 +32,7 @@ class E2EWidgetsFlutterBinding extends LiveTestWidgetsFlutterBinding {
         }
         await _channel.invokeMethod<void>(
           'allTestsFinished',
-          <String, dynamic>{'results': _results},
+          <String, dynamic>{'results': results},
         );
       } on MissingPluginException {
         print('Warning: E2E test plugin was not detected.');
@@ -45,7 +45,7 @@ class E2EWidgetsFlutterBinding extends LiveTestWidgetsFlutterBinding {
     final TestExceptionReporter oldTestExceptionReporter = reportTestException;
     reportTestException =
         (FlutterErrorDetails details, String testDescription) {
-      _results[testDescription] = 'failed';
+      results[testDescription] = 'failed';
       _failureMethodsDetails.add(Failure(testDescription, details.toString()));
       if (!_allTestsPassed.isCompleted) {
         _allTestsPassed.complete(false);
@@ -89,7 +89,12 @@ class E2EWidgetsFlutterBinding extends LiveTestWidgetsFlutterBinding {
 
   static const MethodChannel _channel = MethodChannel('plugins.flutter.io/e2e');
 
-  static Map<String, String> _results = <String, String>{};
+  /// Test results that will be populated after the tests have completed.
+  ///
+  /// Keys are the test descriptions, and values are either `success` or
+  /// `failed`.
+  @visibleForTesting
+  Map<String, String> results = <String, String>{};
 
   /// The extra data for the reported result.
   ///
@@ -153,6 +158,6 @@ class E2EWidgetsFlutterBinding extends LiveTestWidgetsFlutterBinding {
       description: description,
       timeout: timeout,
     );
-    _results[description] ??= 'success';
+    results[description] ??= 'success';
   }
 }

packages/e2e/test/binding_fail_test.dart

@@ -0,0 +1,80 @@
+import 'dart:async';
+import 'dart:io';
+import 'dart:convert';
+
+import 'package:flutter_test/flutter_test.dart';
+
+// Assumes that the flutter command is in `$PATH`.
+const String _flutterBin = 'flutter';
+const String _e2eResultsPrefix = 'E2EWidgetsFlutterBinding test results:';
+
+void main() async {
+  group('E2E binding result', () {
+    test('when multiple tests pass', () async {
+      final Map<String, dynamic> results =
+          await _runTest('test/data/pass_test_script.dart');
+
+      expect(
+          results,
+          equals({
+            'passing test 1': 'success',
+            'passing test 2': 'success',
+          }));
+    });
+
+    test('when multiple tests fail', () async {
+      final Map<String, dynamic> results =
+          await _runTest('test/data/fail_test_script.dart');
+
+      expect(
+          results,
+          equals({
+            'failing test 1': 'failed',
+            'failing test 2': 'failed',
+          }));
+    });
+
+    test('when one test passes, then another fails', () async {
+      final Map<String, dynamic> results =
+          await _runTest('test/data/pass_then_fail_test_script.dart');
+
+      expect(
+          results,
+          equals({
+            'passing test': 'success',
+            'failing test': 'failed',
+          }));
+    });
+  });
+}
+
+/// Runs a test script and returns the [E2EWidgetsFlutterBinding.result].
+///
+/// [scriptPath] is relative to the package root.
+Future<Map<String, dynamic>> _runTest(String scriptPath) async {
+  final Process process =
+      await Process.start(_flutterBin, ['test', '--machine', scriptPath]);
+
+  // In the test [tearDownAll] block, the test results are encoded into JSON and
+  // are printed with the [_e2eResultsPrefix] prefix.
+  //
+  // See the following for the test event spec which we parse the printed lines
+  // out of: https://github.com/dart-lang/test/blob/master/pkgs/test/doc/json_reporter.md
+  final String testResults = (await process.stdout
+          .transform(utf8.decoder)
+          .expand((String text) => text.split('\n'))
+          .map((String line) {
+            try {
+              return jsonDecode(line);
+            } on FormatException {
+              // Only interested in test events which are JSON.
+            }
+          })
+          .where((dynamic testEvent) => testEvent != null && testEvent['type'] == 'print')
+          .map((dynamic printEvent) => printEvent['message'] as String)
+          .firstWhere(
+              (String message) => message.startsWith(_e2eResultsPrefix)))
+      .replaceAll(_e2eResultsPrefix, '');
+
+  return jsonDecode(testResults);
+}

packages/e2e/test/data/README.md

@@ -0,0 +1,4 @@
+Files in this directory are not invoked directly by the test command.
+
+They are used as inputs for the other test files outside of this directory, so
+that failures can be tested.

packages/e2e/test/data/fail_test_script.dart

@@ -0,0 +1,22 @@
+import 'dart:convert';
+
+import 'package:e2e/e2e.dart';
+import 'package:flutter_test/flutter_test.dart';
+
+void main() async {
+  final E2EWidgetsFlutterBinding binding =
+      E2EWidgetsFlutterBinding.ensureInitialized();
+
+  testWidgets('failing test 1', (WidgetTester tester) async {
+    expect(true, false);
+  });
+
+  testWidgets('failing test 2', (WidgetTester tester) async {
+    expect(true, false);
+  });
+
+  tearDownAll(() {
+    print(
+        'E2EWidgetsFlutterBinding test results: ${jsonEncode(binding.results)}');
+  });
+}

packages/e2e/test/data/pass_test_script.dart

@@ -0,0 +1,22 @@
+import 'dart:convert';
+
+import 'package:e2e/e2e.dart';
+import 'package:flutter_test/flutter_test.dart';
+
+void main() async {
+  final E2EWidgetsFlutterBinding binding =
+      E2EWidgetsFlutterBinding.ensureInitialized();
+
+  testWidgets('passing test 1', (WidgetTester tester) async {
+    expect(true, true);
+  });
+
+  testWidgets('passing test 2', (WidgetTester tester) async {
+    expect(true, true);
+  });
+
+  tearDownAll(() {
+    print(
+        'E2EWidgetsFlutterBinding test results: ${jsonEncode(binding.results)}');
+  });
+}

packages/e2e/test/data/pass_then_fail_test_script.dart

@@ -0,0 +1,22 @@
+import 'dart:convert';
+
+import 'package:e2e/e2e.dart';
+import 'package:flutter_test/flutter_test.dart';
+
+void main() async {
+  final E2EWidgetsFlutterBinding binding =
+      E2EWidgetsFlutterBinding.ensureInitialized();
+
+  testWidgets('passing test', (WidgetTester tester) async {
+    expect(true, true);
+  });
+
+  testWidgets('failing test', (WidgetTester tester) async {
+    expect(true, false);
+  });
+
+  tearDownAll(() {
+    print(
+        'E2EWidgetsFlutterBinding test results: ${jsonEncode(binding.results)}');
+  });
+}