diff --git a/lib/web_ui/dev/README.md b/lib/web_ui/dev/README.md index 5e5f5d45c0832..c79dc1f3d6c46 100644 --- a/lib/web_ui/dev/README.md +++ b/lib/web_ui/dev/README.md @@ -31,12 +31,24 @@ felt build [-w] -j 100 If you are a Google employee, you can use an internal instance of Goma to parallelize your builds. Because Goma compiles code on remote servers, this option is effective even on low-powered laptops. ## Running web engine tests -To run all tests on Chrome: +To run all tests on Chrome. This will run both integration tests and the unit tests: ``` felt test ``` +To run unit tests only: + +``` +felt test --unit-tests-only +``` + +To run integration tests only. For now these tests are only available on Chrome Desktop browsers. + +``` +felt test --integration-tests-only +``` + To run tests on Firefox (this will work only on a Linux device): ``` @@ -55,7 +67,7 @@ To run tests on Safari use the following command. It works on MacOS devices and felt test --browser=safari ``` -To run tests on Windows Edge use the following command. It works on Windows devices and it uses the Edge installed on the OS. +To run tests on Windows Edge use the following command. It works on Windows devices and it uses the Edge installed on the OS. ``` felt_windows.bat test --browser=edge diff --git a/lib/web_ui/dev/common.dart b/lib/web_ui/dev/common.dart index 9fa8d126c86ae..37b8e2bf93473 100644 --- a/lib/web_ui/dev/common.dart +++ b/lib/web_ui/dev/common.dart @@ -27,6 +27,15 @@ class BrowserInstallerException implements Exception { String toString() => message; } +class DriverException implements Exception { + DriverException(this.message); + + final String message; + + @override + String toString() => message; +} + abstract class PlatformBinding { static PlatformBinding get instance { if (_instance == null) { @@ -77,11 +86,10 @@ class _WindowsBinding implements PlatformBinding { @override String getFirefoxDownloadUrl(String version) => 'https://download-installer.cdn.mozilla.net/pub/firefox/releases/${version}/win64/en-US/' - '${getFirefoxDownloadFilename(version)}'; + '${getFirefoxDownloadFilename(version)}'; @override - String getFirefoxDownloadFilename(String version) => - 'firefox-${version}.exe'; + String getFirefoxDownloadFilename(String version) => 'firefox-${version}.exe'; @override String getFirefoxExecutablePath(io.Directory versionDir) => @@ -117,7 +125,7 @@ class _LinuxBinding implements PlatformBinding { @override String getFirefoxDownloadUrl(String version) => 'https://download-installer.cdn.mozilla.net/pub/firefox/releases/${version}/linux-x86_64/en-US/' - '${getFirefoxDownloadFilename(version)}'; + '${getFirefoxDownloadFilename(version)}'; @override String getFirefoxDownloadFilename(String version) => @@ -161,16 +169,15 @@ class _MacBinding implements PlatformBinding { @override String getFirefoxDownloadUrl(String version) => - 'https://download-installer.cdn.mozilla.net/pub/firefox/releases/${version}/mac/en-US/' - '${getFirefoxDownloadFilename(version)}'; + 'https://download-installer.cdn.mozilla.net/pub/firefox/releases/${version}/mac/en-US/' + '${getFirefoxDownloadFilename(version)}'; @override - String getFirefoxDownloadFilename(String version) => - 'Firefox ${version}.dmg'; + String getFirefoxDownloadFilename(String version) => 'Firefox ${version}.dmg'; @override String getFirefoxExecutablePath(io.Directory versionDir) => - path.join(versionDir.path, 'Firefox.app','Contents','MacOS', 'firefox'); + path.join(versionDir.path, 'Firefox.app', 'Contents', 'MacOS', 'firefox'); @override String getFirefoxLatestVersionUrl() => @@ -243,3 +250,14 @@ class DevNull implements StringSink { } bool get isCirrus => io.Platform.environment['CIRRUS_CI'] == 'true'; + +/// There might be proccesses started during the tests. +/// +/// Use this list to store those Processes, for cleaning up before shutdown. +final List processesToCleanUp = List(); + +/// There might be temporary directories created during the tests. +/// +/// Use this list to store those directories and for deleteing them before +/// shutdown. +final List temporaryDirectories = List(); diff --git a/lib/web_ui/dev/environment.dart b/lib/web_ui/dev/environment.dart index 35c11a9e073dc..6af6e6e874271 100644 --- a/lib/web_ui/dev/environment.dart +++ b/lib/web_ui/dev/environment.dart @@ -22,6 +22,7 @@ class Environment { final io.Directory hostDebugUnoptDir = io.Directory(pathlib.join(outDir.path, 'host_debug_unopt')); final io.Directory dartSdkDir = io.Directory(pathlib.join(hostDebugUnoptDir.path, 'dart-sdk')); final io.Directory webUiRootDir = io.Directory(pathlib.join(engineSrcDir.path, 'flutter', 'lib', 'web_ui')); + final io.Directory integrationTestsDir = io.Directory(pathlib.join(engineSrcDir.path, 'flutter', 'e2etests', 'web')); for (io.Directory expectedDirectory in [engineSrcDir, outDir, hostDebugUnoptDir, dartSdkDir, webUiRootDir]) { if (!expectedDirectory.existsSync()) { @@ -34,6 +35,7 @@ class Environment { self: self, webUiRootDir: webUiRootDir, engineSrcDir: engineSrcDir, + integrationTestsDir: integrationTestsDir, outDir: outDir, hostDebugUnoptDir: hostDebugUnoptDir, dartSdkDir: dartSdkDir, @@ -44,6 +46,7 @@ class Environment { this.self, this.webUiRootDir, this.engineSrcDir, + this.integrationTestsDir, this.outDir, this.hostDebugUnoptDir, this.dartSdkDir, @@ -58,6 +61,9 @@ class Environment { /// Path to the engine's "src" directory. final io.Directory engineSrcDir; + /// Path to the web integration tests. + final io.Directory integrationTestsDir; + /// Path to the engine's "out" directory. /// /// This is where you'll find the ninja output, such as the Dart SDK. diff --git a/lib/web_ui/dev/felt b/lib/web_ui/dev/felt index 07a71f37e8211..56b031ce0b616 100755 --- a/lib/web_ui/dev/felt +++ b/lib/web_ui/dev/felt @@ -56,9 +56,9 @@ install_deps() { KERNEL_NAME=`uname` if [[ $KERNEL_NAME == *"Darwin"* ]] then - echo "Running on MacOS. Increase the user limits" - ulimit -n 50000 - ulimit -u 4096 + echo "Running on MacOS. Increase the user limits" + ulimit -n 50000 + ulimit -u 4096 fi if [[ "$FELT_USE_SNAPSHOT" == "false" || "$FELT_USE_SNAPSHOT" == "0" ]]; then diff --git a/lib/web_ui/dev/felt.dart b/lib/web_ui/dev/felt.dart index d192431791b39..fd4d2f6c73645 100644 --- a/lib/web_ui/dev/felt.dart +++ b/lib/web_ui/dev/felt.dart @@ -9,6 +9,7 @@ import 'package:args/command_runner.dart'; import 'build.dart'; import 'clean.dart'; +import 'common.dart'; import 'licenses.dart'; import 'test_runner.dart'; @@ -41,12 +42,29 @@ void main(List args) async { io.exit(64); // Exit code 64 indicates a usage error. } catch (e) { rethrow; + } finally { + _cleanup(); } // Sometimes the Dart VM refuses to quit. io.exit(io.exitCode); } +void _cleanup() { + // Cleanup remaining processes if any. + if (processesToCleanUp.length > 0) { + for (io.Process process in processesToCleanUp) { + process.kill(); + } + } + // Delete temporary directories. + if (temporaryDirectories.length > 0) { + for (io.Directory directory in temporaryDirectories) { + directory.deleteSync(recursive: true); + } + } +} + void _listenToShutdownSignals() { io.ProcessSignal.sigint.watch().listen((_) { print('Received SIGINT. Shutting down.'); diff --git a/lib/web_ui/dev/integration_tests_manager.dart b/lib/web_ui/dev/integration_tests_manager.dart new file mode 100644 index 0000000000000..bd97ec878a7ef --- /dev/null +++ b/lib/web_ui/dev/integration_tests_manager.dart @@ -0,0 +1,350 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:io' as io; +import 'package:path/path.dart' as pathlib; +import 'package:web_driver_installer/chrome_driver_installer.dart'; + +import 'common.dart'; +import 'environment.dart'; +import 'utils.dart'; + +class IntegrationTestsManager { + final String _browser; + + /// Installation directory for browser's driver. + /// + /// Always re-install since driver can change frequently. + /// It usually changes with each the browser version changes. + /// A better solution would be installing the browser and the driver at the + /// same time. + // TODO(nurhan): change the web installers to install driver and the browser + // at the same time. + final io.Directory _browserDriverDir; + + /// This is the parent directory for all drivers. + /// + /// This directory is saved to [temporaryDirectories] and deleted before + /// tests shutdown. + final io.Directory _drivers; + + IntegrationTestsManager(this._browser) + : this._browserDriverDir = io.Directory( + pathlib.join(environment.webUiRootDir.path, 'drivers', _browser)), + this._drivers = io.Directory( + pathlib.join(environment.webUiRootDir.path, 'drivers')); + + Future runTests() async { + if (_browser != 'chrome') { + print('WARNING: integration tests are only supported on chrome for now'); + return false; + } else { + await prepareDriver(); + // TODO(nurhan): https://github.com/flutter/flutter/issues/52987 + return await _runTests(); + } + } + + void _cloneWebInstallers() async { + final int exitCode = await runProcess( + 'git', + [ + 'clone', + 'https://github.com/flutter/web_installers.git', + ], + workingDirectory: _browserDriverDir.path, + ); + + if (exitCode != 0) { + io.stderr.writeln('ERROR: ' + 'Failed to clone web installers. Exited with exit code $exitCode'); + throw DriverException('ERROR: ' + 'Failed to clone web installers. Exited with exit code $exitCode'); + } + } + + Future _runPubGet(String workingDirectory) async { + final String executable = isCirrus ? environment.pubExecutable : 'flutter'; + final List arguments = isCirrus + ? [ + 'get', + ] + : [ + 'pub', + 'get', + ]; + final int exitCode = await runProcess( + executable, + arguments, + workingDirectory: workingDirectory, + ); + + if (exitCode != 0) { + io.stderr.writeln( + 'ERROR: Failed to run pub get. Exited with exit code $exitCode'); + return false; + } else { + return true; + } + } + + void _runDriver() async { + final int exitCode = await runProcess( + environment.dartExecutable, + [ + 'lib/web_driver_installer.dart', + '${_browser}driver', + '--install-only', + ], + workingDirectory: pathlib.join( + _browserDriverDir.path, 'web_installers', 'packages', 'web_drivers'), + ); + + if (exitCode != 0) { + io.stderr.writeln( + 'ERROR: Failed to run driver. Exited with exit code $exitCode'); + throw DriverException( + 'ERROR: Failed to run driver. Exited with exit code $exitCode'); + } + startProcess( + './chromedriver/chromedriver', + ['--port=4444'], + workingDirectory: pathlib.join( + _browserDriverDir.path, 'web_installers', 'packages', 'web_drivers'), + ); + print('INFO: Driver started'); + } + + void prepareDriver() async { + final io.Directory priorCurrentDirectory = io.Directory.current; + if (_browserDriverDir.existsSync()) { + _browserDriverDir.deleteSync(recursive: true); + } + + _browserDriverDir.createSync(recursive: true); + temporaryDirectories.add(_drivers); + + // TODO(nurhan): We currently need git clone for getting the driver lock + // file. Remove this after making changes in web_installers. + await _cloneWebInstallers(); + // Change the directory to the driver_lock.yaml file's directory. + io.Directory.current = pathlib.join( + _browserDriverDir.path, 'web_installers', 'packages', 'web_drivers'); + // Chrome is the only browser supporting integration tests for now. + ChromeDriverInstaller chromeDriverInstaller = ChromeDriverInstaller(); + bool installation = await chromeDriverInstaller.install(); + + if (installation) { + io.Directory.current = priorCurrentDirectory; + await _runDriver(); + } else { + throw DriverException('ERROR: Installing driver failed'); + } + } + + /// Runs all the web tests under e2e_tests/web. + Future _runTests() async { + // Only list the files under e2e_tests/web. + final List entities = + environment.integrationTestsDir.listSync(followLinks: false); + + bool allTestsPassed = true; + for (io.FileSystemEntity e in entities) { + // The tests should be under this directories. + if (e is io.Directory) { + allTestsPassed = allTestsPassed && await _validateAndRunTests(e); + } + } + return allTestsPassed; + } + + /// Run tests in a single directory under: e2e_tests/web. + /// + /// Run `flutter pub get` as the first step. + /// + /// Validate the directory before running the tests. Each directory is + /// expected to be a test project which includes a `pubspec.yaml` file + /// and a `test_driver` directory. + Future _validateAndRunTests(io.Directory directory) async { + _validateTestDirectory(directory); + await _runPubGet(directory.path); + final bool testResults = await _runTestsInDirectory(directory); + return testResults; + } + + Future _runTestsInDirectory(io.Directory directory) async { + final io.Directory testDirectory = + io.Directory(pathlib.join(directory.path, 'test_driver')); + final List entities = testDirectory + .listSync(followLinks: false) + .whereType() + .toList(); + + final List e2eTestsToRun = List(); + + // The following loops over the contents of the directory and saves an + // expected driver file name for each e2e test assuming any dart file + // not ending with `_test.dart` is an e2e test. + // Other files are not considered since developers can add files such as + // README. + for (io.File f in entities) { + final String basename = pathlib.basename(f.path); + if (!basename.contains('_test.dart') && basename.endsWith('.dart')) { + e2eTestsToRun.add(basename); + } + } + + print( + 'INFO: In project ${directory} ${e2eTestsToRun.length} tests to run.'); + + int numberOfPassedTests = 0; + int numberOfFailedTests = 0; + for (String fileName in e2eTestsToRun) { + final bool testResults = + await _runTestsInProfileMode(directory, fileName); + if (testResults) { + numberOfPassedTests++; + } else { + numberOfFailedTests++; + } + } + final int numberOfTestsRun = numberOfPassedTests + numberOfFailedTests; + + print('INFO: ${numberOfTestsRun} tests run. ${numberOfPassedTests} passed ' + 'and ${numberOfFailedTests} failed.'); + return numberOfFailedTests == 0; + } + + Future _runTestsInProfileMode( + io.Directory directory, String testName) async { + final int exitCode = await runProcess( + 'flutter', + [ + 'drive', + '--target=test_driver/${testName}', + '-d', + 'web-server', + '--profile', + '--browser-name=$_browser', + '--local-engine=host_debug_unopt', + ], + workingDirectory: directory.path, + ); + + if (exitCode != 0) { + final String statementToRun = 'flutter drive ' + '--target=test_driver/${testName} -d web-server --profile ' + '--browser-name=$_browser --local-engine=host_debug_unopt'; + io.stderr + .writeln('ERROR: Failed to run test. Exited with exit code $exitCode' + '. Statement to run $testName locally use the following ' + 'command:\n\n$statementToRun'); + return false; + } else { + return true; + } + } + + /// Validate the directory has a `pubspec.yaml` file and a `test_driver` + /// directory. + /// + /// Also check the validity of files under `test_driver` directory calling + /// [_checkE2ETestsValidity] method. + void _validateTestDirectory(io.Directory directory) { + final List entities = + directory.listSync(followLinks: false); + + // Whether the project has the pubspec.yaml file. + bool pubSpecFound = false; + // The test directory 'test_driver'. + io.Directory testDirectory = null; + + for (io.FileSystemEntity e in entities) { + // The tests should be under this directories. + final String baseName = pathlib.basename(e.path); + if (e is io.Directory && baseName == 'test_driver') { + testDirectory = e; + } + if (e is io.File && baseName == 'pubspec.yaml') { + pubSpecFound = true; + } + } + if (!pubSpecFound) { + throw StateError('ERROR: pubspec.yaml file not found in the test project ' + 'in the directory ${directory.path}.'); + } + if (testDirectory == null) { + throw StateError( + 'ERROR: test_driver folder not found in the test project.' + 'in the directory ${directory.path}.'); + } else { + _checkE2ETestsValidity(testDirectory); + } + } + + /// Checks if each e2e test file in the directory has a driver test + /// file to run it. + /// + /// Prints informative message to the developer if an error has found. + /// For each e2e test which has name {name}.dart there will be a driver + /// file which drives it. The driver file should be named: + /// {name}_test.dart + void _checkE2ETestsValidity(io.Directory testDirectory) { + final Iterable directories = + testDirectory.listSync(followLinks: false).whereType(); + + if (directories.length > 0) { + throw StateError('${testDirectory.path} directory should not contain ' + 'any sub-directories'); + } + + final Iterable entities = + testDirectory.listSync(followLinks: false).whereType(); + + final Set expectedDriverFileNames = Set(); + final Set foundDriverFileNames = Set(); + int numberOfTests = 0; + + // The following loops over the contents of the directory and saves an + // expected driver file name for each e2e test assuming any file + // not ending with `_test.dart` is an e2e test. + for (io.File f in entities) { + final String basename = pathlib.basename(f.path); + if (basename.contains('_test.dart')) { + // First remove this from expectedSet if not there add to the foundSet. + if (!expectedDriverFileNames.remove(basename)) { + foundDriverFileNames.add(basename); + } + } else if (basename.contains('.dart')) { + // Only run on dart files. + final String e2efileName = pathlib.basenameWithoutExtension(f.path); + final String expectedDriverName = '${e2efileName}_test.dart'; + numberOfTests++; + // First remove this from foundSet if not there add to the expectedSet. + if (!foundDriverFileNames.remove(expectedDriverName)) { + expectedDriverFileNames.add(expectedDriverName); + } + } + } + + if (numberOfTests == 0) { + throw StateError( + 'WARNING: No tests to run in this directory ${testDirectory.path}'); + } + + // TODO(nurhan): In order to reduce the work required from team members, + // remove the need for driver file, by using the same template file. + // Some driver files are missing. + if (expectedDriverFileNames.length > 0) { + for (String expectedDriverName in expectedDriverFileNames) { + print('ERROR: Test driver file named has ${expectedDriverName} ' + 'not found under directory ${testDirectory.path}. Stopping the ' + 'integration tests. Please add ${expectedDriverName}. Check to ' + 'README file on more details on how to setup integration tests.'); + } + throw StateError('Error in test files. Check the logs for ' + 'further instructions'); + } + } +} diff --git a/lib/web_ui/dev/test_runner.dart b/lib/web_ui/dev/test_runner.dart index 0779ad450fad3..fa66b05954d3c 100644 --- a/lib/web_ui/dev/test_runner.dart +++ b/lib/web_ui/dev/test_runner.dart @@ -15,11 +15,24 @@ import 'package:test_api/src/backend/runtime.dart'; // ignore: implementation_im import 'package:test_core/src/executable.dart' as test; // ignore: implementation_imports +import 'integration_tests_manager.dart'; import 'supported_browsers.dart'; import 'test_platform.dart'; import 'environment.dart'; import 'utils.dart'; +/// The type of tests requested by the tool user. +enum TestTypesRequested { + /// For running the unit tests only. + unit, + + /// For running the integration tests only. + integration, + + /// For running both unit and integration tests. + all, +} + class TestCommand extends Command { TestCommand() { argParser @@ -29,6 +42,19 @@ class TestCommand extends Command { 'opportunity to add breakpoints or inspect loaded code before ' 'running the code.', ) + ..addFlag( + 'unit-tests-only', + defaultsTo: false, + help: 'felt test command runs the unit tests and the integration tests ' + 'at the same time. If this flag is set, only run the unit tests.', + ) + ..addFlag( + 'integration-tests-only', + defaultsTo: false, + help: 'felt test command runs the unit tests and the integration tests ' + 'at the same time. If this flag is set, only run the integration ' + 'tests.', + ) ..addFlag( 'update-screenshot-goldens', defaultsTo: false, @@ -54,11 +80,62 @@ class TestCommand extends Command { @override final String description = 'Run tests.'; + TestTypesRequested testTypesRequested = null; + + /// Check the flags to see what type of tests are requested. + TestTypesRequested findTestType() { + if (argResults['unit-tests-only'] && argResults['integration-tests-only']) { + throw ArgumentError('Conflicting arguments: unit-tests-only and ' + 'integration-tests-only are both set'); + } else if (argResults['unit-tests-only']) { + print('Running the unit tests only'); + return TestTypesRequested.unit; + } else if (argResults['integration-tests-only']) { + if (!isChrome) { + throw UnimplementedError( + 'Integration tests are only available on Chrome Desktop for now'); + } + return TestTypesRequested.integration; + } else { + return TestTypesRequested.all; + } + } + @override Future run() async { SupportedBrowsers.instance ..argParsers.forEach((t) => t.parseOptions(argResults)); + // Check the flags to see what type of integration tests are requested. + testTypesRequested = findTestType(); + + switch (testTypesRequested) { + case TestTypesRequested.unit: + return runUnitTests(); + case TestTypesRequested.integration: + return runIntegrationTests(); + case TestTypesRequested.all: + bool integrationTestResult = await runIntegrationTests(); + bool unitTestResult = await runUnitTests(); + if (integrationTestResult != unitTestResult) { + print('Tests run. Integration tests passed: $integrationTestResult ' + 'unit tests passed: $unitTestResult'); + } + return integrationTestResult && unitTestResult; + } + return false; + } + + Future runIntegrationTests() async { + // TODO(nurhan): https://github.com/flutter/flutter/issues/52983 + if (io.Platform.environment['LUCI_CONTEXT'] != null || isCirrus) { + return true; + } + + return IntegrationTestsManager(browser).runTests(); + } + + Future runUnitTests() async { _copyTestFontsIntoWebUi(); await _buildHostPage(); if (io.Platform.isWindows) { @@ -252,18 +329,18 @@ class TestCommand extends Command { Future _buildTests({List targets}) async { List arguments = [ - 'run', - 'build_runner', - 'build', - 'test', - '-o', - 'build', - if (targets != null) - for (FilePath path in targets) ...[ - '--build-filter=${path.relativeToWebUi}.js', - '--build-filter=${path.relativeToWebUi}.browser_test.dart.js', - ], - ]; + 'run', + 'build_runner', + 'build', + 'test', + '-o', + 'build', + if (targets != null) + for (FilePath path in targets) ...[ + '--build-filter=${path.relativeToWebUi}.js', + '--build-filter=${path.relativeToWebUi}.browser_test.dart.js', + ], + ]; final int exitCode = await runProcess( environment.pubExecutable, arguments, diff --git a/lib/web_ui/dev/utils.dart b/lib/web_ui/dev/utils.dart index d60df326b3eea..5b7e6f306d9a5 100644 --- a/lib/web_ui/dev/utils.dart +++ b/lib/web_ui/dev/utils.dart @@ -9,6 +9,7 @@ import 'dart:io' as io; import 'package:meta/meta.dart'; import 'package:path/path.dart' as path; +import 'common.dart'; import 'environment.dart'; class FilePath { @@ -62,6 +63,25 @@ Future runProcess( return exitCode; } +/// Runs [executable]. Do not follow the exit code or the output. +void startProcess( + String executable, + List arguments, { + String workingDirectory, + bool mustSucceed: false, +}) async { + final io.Process process = await io.Process.start( + executable, + arguments, + workingDirectory: workingDirectory, + // Running the process in a system shell for Windows. Otherwise + // the process is not able to get Dart from path. + runInShell: io.Platform.isWindows, + mode: io.ProcessStartMode.inheritStdio, + ); + processesToCleanUp.add(process); +} + /// Runs [executable] and returns its standard output as a string. /// /// If the process fails, throws a [ProcessException]. diff --git a/lib/web_ui/pubspec.yaml b/lib/web_ui/pubspec.yaml index 12e7d8d8f7d7a..997e119224c72 100644 --- a/lib/web_ui/pubspec.yaml +++ b/lib/web_ui/pubspec.yaml @@ -21,3 +21,8 @@ dev_dependencies: watcher: 0.9.7+12 web_engine_tester: path: ../../web_sdk/web_engine_tester + web_driver_installer: + git: + url: git://github.com/flutter/web_installers.git + path: packages/web_drivers/ + ref: dae38d8839cc39f997fb4229f1382680b8758b4f