From 1b60d2fc822a21b36c81cd93d88ffdc9e68d89e6 Mon Sep 17 00:00:00 2001 From: "Ming Lyu (CareF)" Date: Mon, 3 Aug 2020 19:49:24 -0400 Subject: [PATCH 1/6] add new driver and perf --- packages/e2e/CHANGELOG.md | 5 + .../example/test_driver/example_e2e_test.dart | 2 +- packages/e2e/lib/e2e_driver.dart | 75 +++++++- packages/e2e/lib/e2e_perf.dart | 177 ++++++++++++++++++ packages/e2e/pubspec.yaml | 2 +- .../test/frame_timing_summarizer_test.dart | 37 ++++ 6 files changed, 293 insertions(+), 5 deletions(-) create mode 100644 packages/e2e/lib/e2e_perf.dart create mode 100644 packages/e2e/test/frame_timing_summarizer_test.dart diff --git a/packages/e2e/CHANGELOG.md b/packages/e2e/CHANGELOG.md index bc844ce72086..e48b95d099a7 100644 --- a/packages/e2e/CHANGELOG.md +++ b/packages/e2e/CHANGELOG.md @@ -1,3 +1,8 @@ +## 0.6.3 + +* Add customizable `flutter_driver` adaptor. +* Add utilities for tracking frame performance in an e2e test. + ## 0.6.2+1 * Fix incorrect test results when one test passes then another fails diff --git a/packages/e2e/example/test_driver/example_e2e_test.dart b/packages/e2e/example/test_driver/example_e2e_test.dart index 983c3863dea5..cc3ea1572a52 100644 --- a/packages/e2e/example/test_driver/example_e2e_test.dart +++ b/packages/e2e/example/test_driver/example_e2e_test.dart @@ -2,4 +2,4 @@ import 'dart:async'; import 'package:e2e/e2e_driver.dart' as e2e; -Future main() async => e2e.main(); +Future main() async => e2e.e2eDriver(); diff --git a/packages/e2e/lib/e2e_driver.dart b/packages/e2e/lib/e2e_driver.dart index 2e43c5a36e55..ef89d4cbf9da 100644 --- a/packages/e2e/lib/e2e_driver.dart +++ b/packages/e2e/lib/e2e_driver.dart @@ -1,21 +1,90 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + import 'dart:async'; +import 'dart:convert'; import 'dart:io'; -import 'package:e2e/common.dart' as e2e; import 'package:flutter_driver/flutter_driver.dart'; -Future main() async { +import 'package:e2e/common.dart' as e2e; +import 'package:path/path.dart' as path; + +/// This method remains for backword compatibility. +Future main() => e2eDriver(); + +/// Flutter Driver test output directory. +/// +/// Tests should write any output files to this directory. Defaults to the path +/// set in the FLUTTER_TEST_OUTPUTS_DIR environment variable, or `build` if +/// unset. +String testOutputsDirectory = Platform.environment['FLUTTER_TEST_OUTPUTS_DIR'] ?? 'build'; + +/// The callback type to handle [e2e.Response.data] after the test succcess. +typedef ResponseDataCallback = FutureOr Function(Map); + +/// Writes a json-serializable json data to to +/// [testOutputsDirectory]/`testOutputFilename.json`. +/// +/// This is the default `responseDataCallback` in [e2eDriver]. +Future writeResponseData(Map data, { + String testOutputFilename = 'e2e_perf_summary', +}) async { + assert(testOutputFilename != null); + await fs.directory(testOutputsDirectory).create(recursive: true); + final File file = fs.file(path.join( + testOutputsDirectory, + '$testOutputFilename.json' + )); + final String resultString = _encodeJson(data, true); + await file.writeAsString(resultString); +} + +/// Adaptor to run E2E test using `flutter drive`. +/// +/// `timeout` controls the longest time waited before the test ends. +/// It is not necessarily the execution time for the test app: the test may +/// finish sooner than the `timeout`. +/// +/// `responseDataCallback` is the handler for processing [e2e.Response.data]. +/// The default value is `writeResponseData`. +/// +/// To an E2E test `.dart` using `flutter drive`, put a file named +/// `_test.dart` in the app's `test_driver` directory: +/// +/// ```dart +/// import 'dart:async'; +/// +/// import 'package:e2e/e2e_driver.dart' as e2e; +/// +/// Future main() async => e2e.e2eDriver(); +/// +/// ``` +Future e2eDriver({ + Duration timeout = const Duration(minutes: 1), + ResponseDataCallback responseDataCallback = writeResponseData, +}) async { final FlutterDriver driver = await FlutterDriver.connect(); final String jsonResult = - await driver.requestData(null, timeout: const Duration(minutes: 1)); + await driver.requestData(null, timeout: timeout); final e2e.Response response = e2e.Response.fromJson(jsonResult); await driver.close(); if (response.allTestsPassed) { print('All tests passed.'); + await responseDataCallback(response.data); exit(0); } else { print('Failure Details:\n${response.formattedFailureDetails}'); exit(1); } } + +const JsonEncoder _prettyEncoder = JsonEncoder.withIndent(' '); + +String _encodeJson(Map jsonObject, bool pretty) { + return pretty + ? _prettyEncoder.convert(jsonObject) + : json.encode(jsonObject); +} diff --git a/packages/e2e/lib/e2e_perf.dart b/packages/e2e/lib/e2e_perf.dart new file mode 100644 index 000000000000..d7903f0032cd --- /dev/null +++ b/packages/e2e/lib/e2e_perf.dart @@ -0,0 +1,177 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; +import 'dart:ui'; + +import 'package:flutter/scheduler.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:flutter/widgets.dart'; + +import 'package:e2e/e2e.dart'; + +/// The maximum amount of time considered safe to spend for a frame's build +/// phase. Anything past that is in the danger of missing the frame as 60FPS. +/// +/// Changing this doesn't re-evaluate existing summary. +Duration kBuildBudget = const Duration(milliseconds: 16); +// TODO(CareF): Automatically calculate the refresh budget (#61958) + +/// Usually it's recommended to limit callbacks of the test to [WidgetController] +/// API so it can be more universally used. +typedef ControlCallback = Future Function(WidgetController controller); + +bool _firstRun = true; + +/// watches the [FrameTiming] of `action` and report it to the e2e binding. +Future watchPerformance( + E2EWidgetsFlutterBinding binding, + Future action(), +) async { + assert(() { + if (_firstRun) { + debugPrint(kDebugWarning); + _firstRun = false; + } + return true; + }()); + final List frameTimings = []; + final TimingsCallback watcher = frameTimings.addAll; + binding.addTimingsCallback(watcher); + await action(); + binding.removeTimingsCallback(watcher); + // TODO(CareF): determine if it's running on firebase and report metric online + final FrameTimingSummarizer frameTimes = FrameTimingSummarizer(frameTimings); + binding.reportData = {'performance': frameTimes.summary}; +} + +/// This class and summarizes a list of [FrameTiming] for the performance +/// metrics. +class FrameTimingSummarizer { + /// Summarize `data` to frame build time and frame rasterizer time statistics. + /// + /// See [TimelineSummary.summaryJson] for detail. + factory FrameTimingSummarizer(List data) { + assert(data != null); + assert(data.isNotEmpty); + final List frameBuildTime = List.unmodifiable( + data.map((FrameTiming datum) => datum.buildDuration), + ); + final List frameBuildTimeSorted = List.from(frameBuildTime)..sort(); + final List frameRasterizerTime = List.unmodifiable( + data.map((FrameTiming datum) => datum.rasterDuration), + ); + final List frameRasterizerTimeSorted = List.from(frameRasterizerTime)..sort(); + final Duration Function(Duration, Duration) add = (Duration a, Duration b) => a + b; + return FrameTimingSummarizer._( + frameBuildTime: frameBuildTime, + frameRasterizerTime: frameRasterizerTime, + // This avarage calculation is microsecond precision, which is fine + // because typical values of these times are milliseconds. + averageFrameBuildTime: frameBuildTime.reduce(add) ~/ data.length, + p90FrameBuildTime: _findPercentile(frameBuildTimeSorted, 0.90), + p99FrameBuildTime: _findPercentile(frameBuildTimeSorted, 0.99), + worstFrameBuildTime: frameBuildTimeSorted.last, + missedFrameBuildBudget: _countExceed(frameBuildTimeSorted, kBuildBudget), + averageFrameRasterizerTime: frameRasterizerTime.reduce(add) ~/ data.length, + p90FrameRasterizerTime: _findPercentile(frameRasterizerTimeSorted, 0.90), + p99FrameRasterizerTime: _findPercentile(frameRasterizerTimeSorted, 0.99), + worstFrameRasterizerTime: frameRasterizerTimeSorted.last, + missedFrameRasterizerBudget: _countExceed(frameRasterizerTimeSorted, kBuildBudget), + ); + } + + const FrameTimingSummarizer._({ + @required this.frameBuildTime, + @required this.frameRasterizerTime, + @required this.averageFrameBuildTime, + @required this.p90FrameBuildTime, + @required this.p99FrameBuildTime, + @required this.worstFrameBuildTime, + @required this.missedFrameBuildBudget, + @required this.averageFrameRasterizerTime, + @required this.p90FrameRasterizerTime, + @required this.p99FrameRasterizerTime, + @required this.worstFrameRasterizerTime, + @required this.missedFrameRasterizerBudget + }); + + /// List of frame build time in microseconds + final List frameBuildTime; + + /// List of frame rasterizer time in microseconds + final List frameRasterizerTime; + + /// The average value of [frameBuildTime] in milliseconds. + final Duration averageFrameBuildTime; + + /// The 90-th percentile value of [frameBuildTime] in milliseconds + final Duration p90FrameBuildTime; + + /// The 99-th percentile value of [frameBuildTime] in milliseconds + final Duration p99FrameBuildTime; + + /// The largest value of [frameBuildTime] in milliseconds + final Duration worstFrameBuildTime; + + /// Number of items in [frameBuildTime] that's greater than [kBuildBudget] + final int missedFrameBuildBudget; + + /// The average value of [frameRasterizerTime] in milliseconds. + final Duration averageFrameRasterizerTime; + + /// The 90-th percentile value of [frameRasterizerTime] in milliseconds. + final Duration p90FrameRasterizerTime; + + /// The 99-th percentile value of [frameRasterizerTime] in milliseconds. + final Duration p99FrameRasterizerTime; + + /// The largest value of [frameRasterizerTime] in milliseconds. + final Duration worstFrameRasterizerTime; + + /// Number of items in [frameRasterizerTime] that's greater than [kBuildBudget] + final int missedFrameRasterizerBudget; + + /// Convert the summary result to a json object. + /// + /// See [TimelineSummary.summaryJson] for detail. + Map get summary => { + 'average_frame_build_time_millis': + averageFrameBuildTime.inMicroseconds / 1E3, + '90th_percentile_frame_build_time_millis': + p90FrameBuildTime.inMicroseconds / 1E3, + '99th_percentile_frame_build_time_millis': + p99FrameBuildTime.inMicroseconds / 1E3, + 'worst_frame_build_time_millis': + worstFrameBuildTime.inMicroseconds / 1E3, + 'missed_frame_build_budget_count': missedFrameBuildBudget, + 'average_frame_rasterizer_time_millis': + averageFrameRasterizerTime.inMicroseconds / 1E3, + '90th_percentile_frame_rasterizer_time_millis': + p90FrameRasterizerTime.inMicroseconds / 1E3, + '99th_percentile_frame_rasterizer_time_millis': + p99FrameRasterizerTime.inMicroseconds / 1E3, + 'worst_frame_rasterizer_time_millis': + worstFrameRasterizerTime.inMicroseconds / 1E3, + 'missed_frame_rasterizer_budget_count': missedFrameRasterizerBudget, + 'frame_count': frameBuildTime.length, + 'frame_build_times': frameBuildTime + .map((Duration datum) => datum.inMicroseconds).toList(), + 'frame_rasterizer_times': frameRasterizerTime + .map((Duration datum) => datum.inMicroseconds).toList(), + }; +} + +// The following helper functions require data sorted + +// return the 100*p-th percentile of the data +T _findPercentile(List data, double p) { + assert(p >= 0 && p <= 1); + return data[((data.length - 1) * p).round()]; +} + +// return the number of items in data that > threshold +int _countExceed>(List data, T threshold) { + return data.length - data.indexWhere((T datum) => datum.compareTo(threshold) > 0); +} diff --git a/packages/e2e/pubspec.yaml b/packages/e2e/pubspec.yaml index e3f39c05334b..fbe71eeb1584 100644 --- a/packages/e2e/pubspec.yaml +++ b/packages/e2e/pubspec.yaml @@ -1,6 +1,6 @@ name: e2e description: Runs tests that use the flutter_test API as integration tests. -version: 0.6.2+1 +version: 0.6.3 homepage: https://github.com/flutter/plugins/tree/master/packages/e2e environment: diff --git a/packages/e2e/test/frame_timing_summarizer_test.dart b/packages/e2e/test/frame_timing_summarizer_test.dart new file mode 100644 index 000000000000..56577ac528ba --- /dev/null +++ b/packages/e2e/test/frame_timing_summarizer_test.dart @@ -0,0 +1,37 @@ +import 'dart:ui'; + +import 'package:flutter_test/flutter_test.dart'; + +import 'package:e2e/e2e_perf.dart'; + +void main() { + test('Test FrameTimingSummarizer', () { + List buildTimes = [ + for (int i = 1; i <= 100; i += 1) + 1000 * i, + ]; + buildTimes = buildTimes.reversed.toList(); + List rasterTimes = [ + for (int i = 1; i <= 100; i += 1) + 1000 * i + 1000, + ]; + rasterTimes = rasterTimes.reversed.toList(); + List inputData = [ + for (int i = 0; i < 100; i += 1) + FrameTiming([0, buildTimes[i], 500, rasterTimes[i]]), + ]; + FrameTimingSummarizer summary = FrameTimingSummarizer(inputData); + expect(summary.averageFrameBuildTime.inMicroseconds, 50500); + expect(summary.p90FrameBuildTime.inMicroseconds, 90000); + expect(summary.p99FrameBuildTime.inMicroseconds, 99000); + expect(summary.worstFrameBuildTime.inMicroseconds, 100000); + expect(summary.missedFrameBuildBudget, 84); + + expect(summary.averageFrameRasterizerTime.inMicroseconds, 51000); + expect(summary.p90FrameRasterizerTime.inMicroseconds, 90500); + expect(summary.p99FrameRasterizerTime.inMicroseconds, 99500); + expect(summary.worstFrameRasterizerTime.inMicroseconds, 100500); + expect(summary.missedFrameRasterizerBudget, 85); + expect(summary.frameBuildTime.length, 100); + }); +} From d8fffefba677fa9159423bab42f4b4e53dae9342 Mon Sep 17 00:00:00 2001 From: "Ming Lyu (CareF)" Date: Mon, 3 Aug 2020 20:06:10 -0400 Subject: [PATCH 2/6] formatting --- packages/e2e/lib/e2e_driver.dart | 15 ++-- packages/e2e/lib/e2e_perf.dart | 70 +++++++++++-------- packages/e2e/pubspec.yaml | 1 + .../test/frame_timing_summarizer_test.dart | 6 +- 4 files changed, 49 insertions(+), 43 deletions(-) diff --git a/packages/e2e/lib/e2e_driver.dart b/packages/e2e/lib/e2e_driver.dart index ef89d4cbf9da..d0c923c5959f 100644 --- a/packages/e2e/lib/e2e_driver.dart +++ b/packages/e2e/lib/e2e_driver.dart @@ -19,7 +19,8 @@ Future main() => e2eDriver(); /// Tests should write any output files to this directory. Defaults to the path /// set in the FLUTTER_TEST_OUTPUTS_DIR environment variable, or `build` if /// unset. -String testOutputsDirectory = Platform.environment['FLUTTER_TEST_OUTPUTS_DIR'] ?? 'build'; +String testOutputsDirectory = + Platform.environment['FLUTTER_TEST_OUTPUTS_DIR'] ?? 'build'; /// The callback type to handle [e2e.Response.data] after the test succcess. typedef ResponseDataCallback = FutureOr Function(Map); @@ -28,14 +29,15 @@ typedef ResponseDataCallback = FutureOr Function(Map); /// [testOutputsDirectory]/`testOutputFilename.json`. /// /// This is the default `responseDataCallback` in [e2eDriver]. -Future writeResponseData(Map data, { +Future writeResponseData( + Map data, { String testOutputFilename = 'e2e_perf_summary', }) async { assert(testOutputFilename != null); await fs.directory(testOutputsDirectory).create(recursive: true); final File file = fs.file(path.join( testOutputsDirectory, - '$testOutputFilename.json' + '$testOutputFilename.json', )); final String resultString = _encodeJson(data, true); await file.writeAsString(resultString); @@ -66,8 +68,7 @@ Future e2eDriver({ ResponseDataCallback responseDataCallback = writeResponseData, }) async { final FlutterDriver driver = await FlutterDriver.connect(); - final String jsonResult = - await driver.requestData(null, timeout: timeout); + final String jsonResult = await driver.requestData(null, timeout: timeout); final e2e.Response response = e2e.Response.fromJson(jsonResult); await driver.close(); @@ -84,7 +85,5 @@ Future e2eDriver({ const JsonEncoder _prettyEncoder = JsonEncoder.withIndent(' '); String _encodeJson(Map jsonObject, bool pretty) { - return pretty - ? _prettyEncoder.convert(jsonObject) - : json.encode(jsonObject); + return pretty ? _prettyEncoder.convert(jsonObject) : json.encode(jsonObject); } diff --git a/packages/e2e/lib/e2e_perf.dart b/packages/e2e/lib/e2e_perf.dart index d7903f0032cd..4eb5a8d6178e 100644 --- a/packages/e2e/lib/e2e_perf.dart +++ b/packages/e2e/lib/e2e_perf.dart @@ -58,12 +58,15 @@ class FrameTimingSummarizer { final List frameBuildTime = List.unmodifiable( data.map((FrameTiming datum) => datum.buildDuration), ); - final List frameBuildTimeSorted = List.from(frameBuildTime)..sort(); + final List frameBuildTimeSorted = + List.from(frameBuildTime)..sort(); final List frameRasterizerTime = List.unmodifiable( data.map((FrameTiming datum) => datum.rasterDuration), ); - final List frameRasterizerTimeSorted = List.from(frameRasterizerTime)..sort(); - final Duration Function(Duration, Duration) add = (Duration a, Duration b) => a + b; + final List frameRasterizerTimeSorted = + List.from(frameRasterizerTime)..sort(); + final Duration Function(Duration, Duration) add = + (Duration a, Duration b) => a + b; return FrameTimingSummarizer._( frameBuildTime: frameBuildTime, frameRasterizerTime: frameRasterizerTime, @@ -74,11 +77,13 @@ class FrameTimingSummarizer { p99FrameBuildTime: _findPercentile(frameBuildTimeSorted, 0.99), worstFrameBuildTime: frameBuildTimeSorted.last, missedFrameBuildBudget: _countExceed(frameBuildTimeSorted, kBuildBudget), - averageFrameRasterizerTime: frameRasterizerTime.reduce(add) ~/ data.length, + averageFrameRasterizerTime: + frameRasterizerTime.reduce(add) ~/ data.length, p90FrameRasterizerTime: _findPercentile(frameRasterizerTimeSorted, 0.90), p99FrameRasterizerTime: _findPercentile(frameRasterizerTimeSorted, 0.99), worstFrameRasterizerTime: frameRasterizerTimeSorted.last, - missedFrameRasterizerBudget: _countExceed(frameRasterizerTimeSorted, kBuildBudget), + missedFrameRasterizerBudget: + _countExceed(frameRasterizerTimeSorted, kBuildBudget), ); } @@ -94,7 +99,7 @@ class FrameTimingSummarizer { @required this.p90FrameRasterizerTime, @required this.p99FrameRasterizerTime, @required this.worstFrameRasterizerTime, - @required this.missedFrameRasterizerBudget + @required this.missedFrameRasterizerBudget, }); /// List of frame build time in microseconds @@ -137,30 +142,32 @@ class FrameTimingSummarizer { /// /// See [TimelineSummary.summaryJson] for detail. Map get summary => { - 'average_frame_build_time_millis': - averageFrameBuildTime.inMicroseconds / 1E3, - '90th_percentile_frame_build_time_millis': - p90FrameBuildTime.inMicroseconds / 1E3, - '99th_percentile_frame_build_time_millis': - p99FrameBuildTime.inMicroseconds / 1E3, - 'worst_frame_build_time_millis': - worstFrameBuildTime.inMicroseconds / 1E3, - 'missed_frame_build_budget_count': missedFrameBuildBudget, - 'average_frame_rasterizer_time_millis': - averageFrameRasterizerTime.inMicroseconds / 1E3, - '90th_percentile_frame_rasterizer_time_millis': - p90FrameRasterizerTime.inMicroseconds / 1E3, - '99th_percentile_frame_rasterizer_time_millis': - p99FrameRasterizerTime.inMicroseconds / 1E3, - 'worst_frame_rasterizer_time_millis': - worstFrameRasterizerTime.inMicroseconds / 1E3, - 'missed_frame_rasterizer_budget_count': missedFrameRasterizerBudget, - 'frame_count': frameBuildTime.length, - 'frame_build_times': frameBuildTime - .map((Duration datum) => datum.inMicroseconds).toList(), - 'frame_rasterizer_times': frameRasterizerTime - .map((Duration datum) => datum.inMicroseconds).toList(), - }; + 'average_frame_build_time_millis': + averageFrameBuildTime.inMicroseconds / 1E3, + '90th_percentile_frame_build_time_millis': + p90FrameBuildTime.inMicroseconds / 1E3, + '99th_percentile_frame_build_time_millis': + p99FrameBuildTime.inMicroseconds / 1E3, + 'worst_frame_build_time_millis': + worstFrameBuildTime.inMicroseconds / 1E3, + 'missed_frame_build_budget_count': missedFrameBuildBudget, + 'average_frame_rasterizer_time_millis': + averageFrameRasterizerTime.inMicroseconds / 1E3, + '90th_percentile_frame_rasterizer_time_millis': + p90FrameRasterizerTime.inMicroseconds / 1E3, + '99th_percentile_frame_rasterizer_time_millis': + p99FrameRasterizerTime.inMicroseconds / 1E3, + 'worst_frame_rasterizer_time_millis': + worstFrameRasterizerTime.inMicroseconds / 1E3, + 'missed_frame_rasterizer_budget_count': missedFrameRasterizerBudget, + 'frame_count': frameBuildTime.length, + 'frame_build_times': frameBuildTime + .map((Duration datum) => datum.inMicroseconds) + .toList(), + 'frame_rasterizer_times': frameRasterizerTime + .map((Duration datum) => datum.inMicroseconds) + .toList(), + }; } // The following helper functions require data sorted @@ -173,5 +180,6 @@ T _findPercentile(List data, double p) { // return the number of items in data that > threshold int _countExceed>(List data, T threshold) { - return data.length - data.indexWhere((T datum) => datum.compareTo(threshold) > 0); + return data.length - + data.indexWhere((T datum) => datum.compareTo(threshold) > 0); } diff --git a/packages/e2e/pubspec.yaml b/packages/e2e/pubspec.yaml index fbe71eeb1584..b83acbd28ea8 100644 --- a/packages/e2e/pubspec.yaml +++ b/packages/e2e/pubspec.yaml @@ -14,6 +14,7 @@ dependencies: sdk: flutter flutter_test: sdk: flutter + path: dev_dependencies: pedantic: ^1.8.0 diff --git a/packages/e2e/test/frame_timing_summarizer_test.dart b/packages/e2e/test/frame_timing_summarizer_test.dart index 56577ac528ba..f3a12850122d 100644 --- a/packages/e2e/test/frame_timing_summarizer_test.dart +++ b/packages/e2e/test/frame_timing_summarizer_test.dart @@ -7,13 +7,11 @@ import 'package:e2e/e2e_perf.dart'; void main() { test('Test FrameTimingSummarizer', () { List buildTimes = [ - for (int i = 1; i <= 100; i += 1) - 1000 * i, + for (int i = 1; i <= 100; i += 1) 1000 * i, ]; buildTimes = buildTimes.reversed.toList(); List rasterTimes = [ - for (int i = 1; i <= 100; i += 1) - 1000 * i + 1000, + for (int i = 1; i <= 100; i += 1) 1000 * i + 1000, ]; rasterTimes = rasterTimes.reversed.toList(); List inputData = [ From 72f6169a7dd2761f327b256e291b866d6709bea9 Mon Sep 17 00:00:00 2001 From: "Ming Lyu (CareF)" Date: Mon, 3 Aug 2020 21:01:41 -0400 Subject: [PATCH 3/6] versioning --- packages/e2e/lib/e2e_perf.dart | 18 ++++++++++++++++++ packages/e2e/pubspec.yaml | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/packages/e2e/lib/e2e_perf.dart b/packages/e2e/lib/e2e_perf.dart index 4eb5a8d6178e..d26fb8981cf4 100644 --- a/packages/e2e/lib/e2e_perf.dart +++ b/packages/e2e/lib/e2e_perf.dart @@ -24,6 +24,24 @@ typedef ControlCallback = Future Function(WidgetController controller); bool _firstRun = true; +/// The warning message to show when a benchmark is performed with assert on. +/// TODO(CareF) remove this and update pubspect when flutter/flutter#61509 is +/// in released version. +const String kDebugWarning = ''' +┏╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍┓ +┇ ⚠ THIS BENCHMARK IS BEING RUN IN DEBUG MODE ⚠ ┇ +┡╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍┦ +│ │ +│ Numbers obtained from a benchmark while asserts are │ +│ enabled will not accurately reflect the performance │ +│ that will be experienced by end users using release ╎ +│ builds. Benchmarks should be run using this command ╎ +│ line: "flutter run --profile test.dart" or ┊ +│ or "flutter drive --profile -t test.dart". ┊ +│ ┊ +└─────────────────────────────────────────────────╌┄┈ 🐢 +'''; + /// watches the [FrameTiming] of `action` and report it to the e2e binding. Future watchPerformance( E2EWidgetsFlutterBinding binding, diff --git a/packages/e2e/pubspec.yaml b/packages/e2e/pubspec.yaml index b83acbd28ea8..e4e476d146d3 100644 --- a/packages/e2e/pubspec.yaml +++ b/packages/e2e/pubspec.yaml @@ -14,7 +14,7 @@ dependencies: sdk: flutter flutter_test: sdk: flutter - path: + path: ^1.6.4 dev_dependencies: pedantic: ^1.8.0 From 99533081dfda6e07d00271c2b182751c24ff6ff9 Mon Sep 17 00:00:00 2001 From: "Ming Lyu (CareF)" Date: Tue, 4 Aug 2020 22:43:22 -0400 Subject: [PATCH 4/6] modify according to AlbertWang0116@ --- packages/e2e/lib/e2e_perf.dart | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/e2e/lib/e2e_perf.dart b/packages/e2e/lib/e2e_perf.dart index d26fb8981cf4..a8ddb939ce45 100644 --- a/packages/e2e/lib/e2e_perf.dart +++ b/packages/e2e/lib/e2e_perf.dart @@ -18,10 +18,6 @@ import 'package:e2e/e2e.dart'; Duration kBuildBudget = const Duration(milliseconds: 16); // TODO(CareF): Automatically calculate the refresh budget (#61958) -/// Usually it's recommended to limit callbacks of the test to [WidgetController] -/// API so it can be more universally used. -typedef ControlCallback = Future Function(WidgetController controller); - bool _firstRun = true; /// The warning message to show when a benchmark is performed with assert on. From 21f550a8e6725b4e851376ff76d6ad6036a1dc15 Mon Sep 17 00:00:00 2001 From: "Ming Lyu (CareF)" Date: Wed, 5 Aug 2020 11:16:05 -0400 Subject: [PATCH 5/6] modify according to liyuqian@ --- packages/e2e/lib/e2e_driver.dart | 12 ++++++++---- packages/e2e/lib/e2e_perf.dart | 8 ++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/packages/e2e/lib/e2e_driver.dart b/packages/e2e/lib/e2e_driver.dart index d0c923c5959f..c33083e07574 100644 --- a/packages/e2e/lib/e2e_driver.dart +++ b/packages/e2e/lib/e2e_driver.dart @@ -31,12 +31,14 @@ typedef ResponseDataCallback = FutureOr Function(Map); /// This is the default `responseDataCallback` in [e2eDriver]. Future writeResponseData( Map data, { - String testOutputFilename = 'e2e_perf_summary', + String testOutputFilename = 'e2e_response_data', + String destinationDirectory, }) async { assert(testOutputFilename != null); - await fs.directory(testOutputsDirectory).create(recursive: true); + destinationDirectory ??= testOutputsDirectory; + await fs.directory(destinationDirectory).create(recursive: true); final File file = fs.file(path.join( - testOutputsDirectory, + destinationDirectory, '$testOutputFilename.json', )); final String resultString = _encodeJson(data, true); @@ -74,7 +76,9 @@ Future e2eDriver({ if (response.allTestsPassed) { print('All tests passed.'); - await responseDataCallback(response.data); + if (responseDataCallback != null) { + await responseDataCallback(response.data); + } exit(0); } else { print('Failure Details:\n${response.formattedFailureDetails}'); diff --git a/packages/e2e/lib/e2e_perf.dart b/packages/e2e/lib/e2e_perf.dart index a8ddb939ce45..865068c7f7bc 100644 --- a/packages/e2e/lib/e2e_perf.dart +++ b/packages/e2e/lib/e2e_perf.dart @@ -41,8 +41,9 @@ const String kDebugWarning = ''' /// watches the [FrameTiming] of `action` and report it to the e2e binding. Future watchPerformance( E2EWidgetsFlutterBinding binding, - Future action(), -) async { + Future action(), { + String metricName = 'performance', +}) async { assert(() { if (_firstRun) { debugPrint(kDebugWarning); @@ -55,9 +56,8 @@ Future watchPerformance( binding.addTimingsCallback(watcher); await action(); binding.removeTimingsCallback(watcher); - // TODO(CareF): determine if it's running on firebase and report metric online final FrameTimingSummarizer frameTimes = FrameTimingSummarizer(frameTimings); - binding.reportData = {'performance': frameTimes.summary}; + binding.reportData = {metricName: frameTimes.summary}; } /// This class and summarizes a list of [FrameTiming] for the performance From 7dfaf1e6bc847e98d543ea106809fafd04300309 Mon Sep 17 00:00:00 2001 From: "Ming Lyu (CareF)" Date: Wed, 5 Aug 2020 14:15:00 -0400 Subject: [PATCH 6/6] metricName -> reportKey --- packages/e2e/lib/e2e_perf.dart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/e2e/lib/e2e_perf.dart b/packages/e2e/lib/e2e_perf.dart index 865068c7f7bc..1b2ddd74c336 100644 --- a/packages/e2e/lib/e2e_perf.dart +++ b/packages/e2e/lib/e2e_perf.dart @@ -42,7 +42,7 @@ const String kDebugWarning = ''' Future watchPerformance( E2EWidgetsFlutterBinding binding, Future action(), { - String metricName = 'performance', + String reportKey = 'performance', }) async { assert(() { if (_firstRun) { @@ -57,7 +57,7 @@ Future watchPerformance( await action(); binding.removeTimingsCallback(watcher); final FrameTimingSummarizer frameTimes = FrameTimingSummarizer(frameTimings); - binding.reportData = {metricName: frameTimes.summary}; + binding.reportData = {reportKey: frameTimes.summary}; } /// This class and summarizes a list of [FrameTiming] for the performance