diff --git a/packages/e2e/CHANGELOG.md b/packages/e2e/CHANGELOG.md index bc844ce72086..e48b95d099a7 100644 --- a/packages/e2e/CHANGELOG.md +++ b/packages/e2e/CHANGELOG.md @@ -1,3 +1,8 @@ +## 0.6.3 + +* Add customizable `flutter_driver` adaptor. +* Add utilities for tracking frame performance in an e2e test. + ## 0.6.2+1 * Fix incorrect test results when one test passes then another fails diff --git a/packages/e2e/example/test_driver/example_e2e_test.dart b/packages/e2e/example/test_driver/example_e2e_test.dart index 983c3863dea5..cc3ea1572a52 100644 --- a/packages/e2e/example/test_driver/example_e2e_test.dart +++ b/packages/e2e/example/test_driver/example_e2e_test.dart @@ -2,4 +2,4 @@ import 'dart:async'; import 'package:e2e/e2e_driver.dart' as e2e; -Future main() async => e2e.main(); +Future main() async => e2e.e2eDriver(); diff --git a/packages/e2e/lib/e2e_driver.dart b/packages/e2e/lib/e2e_driver.dart index 2e43c5a36e55..c33083e07574 100644 --- a/packages/e2e/lib/e2e_driver.dart +++ b/packages/e2e/lib/e2e_driver.dart @@ -1,21 +1,93 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + import 'dart:async'; +import 'dart:convert'; import 'dart:io'; -import 'package:e2e/common.dart' as e2e; import 'package:flutter_driver/flutter_driver.dart'; -Future main() async { +import 'package:e2e/common.dart' as e2e; +import 'package:path/path.dart' as path; + +/// This method remains for backword compatibility. +Future main() => e2eDriver(); + +/// Flutter Driver test output directory. +/// +/// Tests should write any output files to this directory. Defaults to the path +/// set in the FLUTTER_TEST_OUTPUTS_DIR environment variable, or `build` if +/// unset. +String testOutputsDirectory = + Platform.environment['FLUTTER_TEST_OUTPUTS_DIR'] ?? 'build'; + +/// The callback type to handle [e2e.Response.data] after the test succcess. +typedef ResponseDataCallback = FutureOr Function(Map); + +/// Writes a json-serializable json data to to +/// [testOutputsDirectory]/`testOutputFilename.json`. +/// +/// This is the default `responseDataCallback` in [e2eDriver]. +Future writeResponseData( + Map data, { + String testOutputFilename = 'e2e_response_data', + String destinationDirectory, +}) async { + assert(testOutputFilename != null); + destinationDirectory ??= testOutputsDirectory; + await fs.directory(destinationDirectory).create(recursive: true); + final File file = fs.file(path.join( + destinationDirectory, + '$testOutputFilename.json', + )); + final String resultString = _encodeJson(data, true); + await file.writeAsString(resultString); +} + +/// Adaptor to run E2E test using `flutter drive`. +/// +/// `timeout` controls the longest time waited before the test ends. +/// It is not necessarily the execution time for the test app: the test may +/// finish sooner than the `timeout`. +/// +/// `responseDataCallback` is the handler for processing [e2e.Response.data]. +/// The default value is `writeResponseData`. +/// +/// To an E2E test `.dart` using `flutter drive`, put a file named +/// `_test.dart` in the app's `test_driver` directory: +/// +/// ```dart +/// import 'dart:async'; +/// +/// import 'package:e2e/e2e_driver.dart' as e2e; +/// +/// Future main() async => e2e.e2eDriver(); +/// +/// ``` +Future e2eDriver({ + Duration timeout = const Duration(minutes: 1), + ResponseDataCallback responseDataCallback = writeResponseData, +}) async { final FlutterDriver driver = await FlutterDriver.connect(); - final String jsonResult = - await driver.requestData(null, timeout: const Duration(minutes: 1)); + final String jsonResult = await driver.requestData(null, timeout: timeout); final e2e.Response response = e2e.Response.fromJson(jsonResult); await driver.close(); if (response.allTestsPassed) { print('All tests passed.'); + if (responseDataCallback != null) { + await responseDataCallback(response.data); + } exit(0); } else { print('Failure Details:\n${response.formattedFailureDetails}'); exit(1); } } + +const JsonEncoder _prettyEncoder = JsonEncoder.withIndent(' '); + +String _encodeJson(Map jsonObject, bool pretty) { + return pretty ? _prettyEncoder.convert(jsonObject) : json.encode(jsonObject); +} diff --git a/packages/e2e/lib/e2e_perf.dart b/packages/e2e/lib/e2e_perf.dart new file mode 100644 index 000000000000..1b2ddd74c336 --- /dev/null +++ b/packages/e2e/lib/e2e_perf.dart @@ -0,0 +1,199 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; +import 'dart:ui'; + +import 'package:flutter/scheduler.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:flutter/widgets.dart'; + +import 'package:e2e/e2e.dart'; + +/// The maximum amount of time considered safe to spend for a frame's build +/// phase. Anything past that is in the danger of missing the frame as 60FPS. +/// +/// Changing this doesn't re-evaluate existing summary. +Duration kBuildBudget = const Duration(milliseconds: 16); +// TODO(CareF): Automatically calculate the refresh budget (#61958) + +bool _firstRun = true; + +/// The warning message to show when a benchmark is performed with assert on. +/// TODO(CareF) remove this and update pubspect when flutter/flutter#61509 is +/// in released version. +const String kDebugWarning = ''' +┏╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍┓ +┇ ⚠ THIS BENCHMARK IS BEING RUN IN DEBUG MODE ⚠ ┇ +┡╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍┦ +│ │ +│ Numbers obtained from a benchmark while asserts are │ +│ enabled will not accurately reflect the performance │ +│ that will be experienced by end users using release ╎ +│ builds. Benchmarks should be run using this command ╎ +│ line: "flutter run --profile test.dart" or ┊ +│ or "flutter drive --profile -t test.dart". ┊ +│ ┊ +└─────────────────────────────────────────────────╌┄┈ 🐢 +'''; + +/// watches the [FrameTiming] of `action` and report it to the e2e binding. +Future watchPerformance( + E2EWidgetsFlutterBinding binding, + Future action(), { + String reportKey = 'performance', +}) async { + assert(() { + if (_firstRun) { + debugPrint(kDebugWarning); + _firstRun = false; + } + return true; + }()); + final List frameTimings = []; + final TimingsCallback watcher = frameTimings.addAll; + binding.addTimingsCallback(watcher); + await action(); + binding.removeTimingsCallback(watcher); + final FrameTimingSummarizer frameTimes = FrameTimingSummarizer(frameTimings); + binding.reportData = {reportKey: frameTimes.summary}; +} + +/// This class and summarizes a list of [FrameTiming] for the performance +/// metrics. +class FrameTimingSummarizer { + /// Summarize `data` to frame build time and frame rasterizer time statistics. + /// + /// See [TimelineSummary.summaryJson] for detail. + factory FrameTimingSummarizer(List data) { + assert(data != null); + assert(data.isNotEmpty); + final List frameBuildTime = List.unmodifiable( + data.map((FrameTiming datum) => datum.buildDuration), + ); + final List frameBuildTimeSorted = + List.from(frameBuildTime)..sort(); + final List frameRasterizerTime = List.unmodifiable( + data.map((FrameTiming datum) => datum.rasterDuration), + ); + final List frameRasterizerTimeSorted = + List.from(frameRasterizerTime)..sort(); + final Duration Function(Duration, Duration) add = + (Duration a, Duration b) => a + b; + return FrameTimingSummarizer._( + frameBuildTime: frameBuildTime, + frameRasterizerTime: frameRasterizerTime, + // This avarage calculation is microsecond precision, which is fine + // because typical values of these times are milliseconds. + averageFrameBuildTime: frameBuildTime.reduce(add) ~/ data.length, + p90FrameBuildTime: _findPercentile(frameBuildTimeSorted, 0.90), + p99FrameBuildTime: _findPercentile(frameBuildTimeSorted, 0.99), + worstFrameBuildTime: frameBuildTimeSorted.last, + missedFrameBuildBudget: _countExceed(frameBuildTimeSorted, kBuildBudget), + averageFrameRasterizerTime: + frameRasterizerTime.reduce(add) ~/ data.length, + p90FrameRasterizerTime: _findPercentile(frameRasterizerTimeSorted, 0.90), + p99FrameRasterizerTime: _findPercentile(frameRasterizerTimeSorted, 0.99), + worstFrameRasterizerTime: frameRasterizerTimeSorted.last, + missedFrameRasterizerBudget: + _countExceed(frameRasterizerTimeSorted, kBuildBudget), + ); + } + + const FrameTimingSummarizer._({ + @required this.frameBuildTime, + @required this.frameRasterizerTime, + @required this.averageFrameBuildTime, + @required this.p90FrameBuildTime, + @required this.p99FrameBuildTime, + @required this.worstFrameBuildTime, + @required this.missedFrameBuildBudget, + @required this.averageFrameRasterizerTime, + @required this.p90FrameRasterizerTime, + @required this.p99FrameRasterizerTime, + @required this.worstFrameRasterizerTime, + @required this.missedFrameRasterizerBudget, + }); + + /// List of frame build time in microseconds + final List frameBuildTime; + + /// List of frame rasterizer time in microseconds + final List frameRasterizerTime; + + /// The average value of [frameBuildTime] in milliseconds. + final Duration averageFrameBuildTime; + + /// The 90-th percentile value of [frameBuildTime] in milliseconds + final Duration p90FrameBuildTime; + + /// The 99-th percentile value of [frameBuildTime] in milliseconds + final Duration p99FrameBuildTime; + + /// The largest value of [frameBuildTime] in milliseconds + final Duration worstFrameBuildTime; + + /// Number of items in [frameBuildTime] that's greater than [kBuildBudget] + final int missedFrameBuildBudget; + + /// The average value of [frameRasterizerTime] in milliseconds. + final Duration averageFrameRasterizerTime; + + /// The 90-th percentile value of [frameRasterizerTime] in milliseconds. + final Duration p90FrameRasterizerTime; + + /// The 99-th percentile value of [frameRasterizerTime] in milliseconds. + final Duration p99FrameRasterizerTime; + + /// The largest value of [frameRasterizerTime] in milliseconds. + final Duration worstFrameRasterizerTime; + + /// Number of items in [frameRasterizerTime] that's greater than [kBuildBudget] + final int missedFrameRasterizerBudget; + + /// Convert the summary result to a json object. + /// + /// See [TimelineSummary.summaryJson] for detail. + Map get summary => { + 'average_frame_build_time_millis': + averageFrameBuildTime.inMicroseconds / 1E3, + '90th_percentile_frame_build_time_millis': + p90FrameBuildTime.inMicroseconds / 1E3, + '99th_percentile_frame_build_time_millis': + p99FrameBuildTime.inMicroseconds / 1E3, + 'worst_frame_build_time_millis': + worstFrameBuildTime.inMicroseconds / 1E3, + 'missed_frame_build_budget_count': missedFrameBuildBudget, + 'average_frame_rasterizer_time_millis': + averageFrameRasterizerTime.inMicroseconds / 1E3, + '90th_percentile_frame_rasterizer_time_millis': + p90FrameRasterizerTime.inMicroseconds / 1E3, + '99th_percentile_frame_rasterizer_time_millis': + p99FrameRasterizerTime.inMicroseconds / 1E3, + 'worst_frame_rasterizer_time_millis': + worstFrameRasterizerTime.inMicroseconds / 1E3, + 'missed_frame_rasterizer_budget_count': missedFrameRasterizerBudget, + 'frame_count': frameBuildTime.length, + 'frame_build_times': frameBuildTime + .map((Duration datum) => datum.inMicroseconds) + .toList(), + 'frame_rasterizer_times': frameRasterizerTime + .map((Duration datum) => datum.inMicroseconds) + .toList(), + }; +} + +// The following helper functions require data sorted + +// return the 100*p-th percentile of the data +T _findPercentile(List data, double p) { + assert(p >= 0 && p <= 1); + return data[((data.length - 1) * p).round()]; +} + +// return the number of items in data that > threshold +int _countExceed>(List data, T threshold) { + return data.length - + data.indexWhere((T datum) => datum.compareTo(threshold) > 0); +} diff --git a/packages/e2e/pubspec.yaml b/packages/e2e/pubspec.yaml index e3f39c05334b..e4e476d146d3 100644 --- a/packages/e2e/pubspec.yaml +++ b/packages/e2e/pubspec.yaml @@ -1,6 +1,6 @@ name: e2e description: Runs tests that use the flutter_test API as integration tests. -version: 0.6.2+1 +version: 0.6.3 homepage: https://github.com/flutter/plugins/tree/master/packages/e2e environment: @@ -14,6 +14,7 @@ dependencies: sdk: flutter flutter_test: sdk: flutter + path: ^1.6.4 dev_dependencies: pedantic: ^1.8.0 diff --git a/packages/e2e/test/frame_timing_summarizer_test.dart b/packages/e2e/test/frame_timing_summarizer_test.dart new file mode 100644 index 000000000000..f3a12850122d --- /dev/null +++ b/packages/e2e/test/frame_timing_summarizer_test.dart @@ -0,0 +1,35 @@ +import 'dart:ui'; + +import 'package:flutter_test/flutter_test.dart'; + +import 'package:e2e/e2e_perf.dart'; + +void main() { + test('Test FrameTimingSummarizer', () { + List buildTimes = [ + for (int i = 1; i <= 100; i += 1) 1000 * i, + ]; + buildTimes = buildTimes.reversed.toList(); + List rasterTimes = [ + for (int i = 1; i <= 100; i += 1) 1000 * i + 1000, + ]; + rasterTimes = rasterTimes.reversed.toList(); + List inputData = [ + for (int i = 0; i < 100; i += 1) + FrameTiming([0, buildTimes[i], 500, rasterTimes[i]]), + ]; + FrameTimingSummarizer summary = FrameTimingSummarizer(inputData); + expect(summary.averageFrameBuildTime.inMicroseconds, 50500); + expect(summary.p90FrameBuildTime.inMicroseconds, 90000); + expect(summary.p99FrameBuildTime.inMicroseconds, 99000); + expect(summary.worstFrameBuildTime.inMicroseconds, 100000); + expect(summary.missedFrameBuildBudget, 84); + + expect(summary.averageFrameRasterizerTime.inMicroseconds, 51000); + expect(summary.p90FrameRasterizerTime.inMicroseconds, 90500); + expect(summary.p99FrameRasterizerTime.inMicroseconds, 99500); + expect(summary.worstFrameRasterizerTime.inMicroseconds, 100500); + expect(summary.missedFrameRasterizerBudget, 85); + expect(summary.frameBuildTime.length, 100); + }); +}