Reland "Use batch mode compilation for -cdartkp --strong"

Preliminary testing shows, this increases performance by around 20%. The
main benefit is by re-using a warmed-up VM and not start one from
scratch for every compilation.

Going forward we can do more optimizations, e.g. reading the platform
dill file only once into memory (instead of repeatedly) ...

  => This requires us using the new state-full IKG compiler.

Issue #31585

Change-Id: I2d3448783fc118611baf4671187a897227a65a6c
Reviewed-on: https://dart-review.googlesource.com/28400
Reviewed-by: Vyacheslav Egorov <[email protected]>

Author: mkustermann

pkg/vm/bin/gen_kernel.dart

@@ -2,12 +2,14 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
+import 'dart:async';
 import 'dart:io';
 
 import 'package:args/args.dart' show ArgParser, ArgResults;
 import 'package:front_end/src/api_prototype/front_end.dart';
 import 'package:kernel/binary/ast_to_binary.dart';
 import 'package:kernel/kernel.dart' show Program;
+import 'package:kernel/src/tool/batch_util.dart' as batch_util;
 import 'package:kernel/target/targets.dart' show TargetFlags;
 import 'package:kernel/target/vm.dart' show VmTarget;
 import 'package:vm/kernel_front_end.dart' show compileToKernel;
@@ -43,12 +45,20 @@ const _severityCaptions = const <Severity, String>{
 };
 
 main(List<String> arguments) async {
+  if (arguments.isNotEmpty && arguments.last == '--batch') {
+    await runBatchModeCompiler();
+  } else {
+    exit(await compile(arguments));
+  }
+}
+
+Future<int> compile(List<String> arguments) async {
   final ArgResults options = _argParser.parse(arguments);
   final String platformKernel = options['platform'];
 
   if ((options.rest.length != 1) || (platformKernel == null)) {
     print(_usage);
-    exit(_badUsageExitCode);
+    return _badUsageExitCode;
   }
 
   final String filename = options.rest.single;
@@ -82,11 +92,41 @@ main(List<String> arguments) async {
       aot: aot);
 
   if ((errors > 0) || (program == null)) {
-    exit(_compileTimeErrorExitCode);
+    return _compileTimeErrorExitCode;
   }
 
   final IOSink sink = new File(kernelBinaryFilename).openWrite();
   final BinaryPrinter printer = new BinaryPrinter(sink);
   printer.writeProgramFile(program);
   await sink.close();
+
+  return 0;
+}
+
+Future runBatchModeCompiler() async {
+  await batch_util.runBatch((List<String> arguments) async {
+    // TODO(kustermann): Once we know where the new IKG api is and how to use
+    // it, we should take advantage of it.
+    //
+    // Important things to note:
+    //
+    //   * Our global transformations must never alter the AST structures which
+    //     the statefull IKG generator keeps across compilations.
+    //     => We need to make our own copy.
+    //
+    //   * We must ensure the stateful IKG generator keeps giving us all the
+    //     compile-time errors, warnings, hints for every compilation and we
+    //     report the compilation result accordingly.
+    //
+    final exitCode = await compile(arguments);
+    switch (exitCode) {
+      case 0:
+        return batch_util.CompilerOutcome.Ok;
+      case _compileTimeErrorExitCode:
+      case _badUsageExitCode:
+        return batch_util.CompilerOutcome.Fail;
+      default:
+        throw 'Could not obtain correct exit code from compiler.';
+    }
+  });
 }

pkg/vm/tool/gen_kernel

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# Copyright (c) 2017, the Dart project authors.  Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+# Script for generating kernel files using Dart 2 pipeline: Fasta with
+# strong mode enabled.
+
+set -e
+
+function follow_links() {
+  file="$1"
+  while [ -h "$file" ]; do
+    # On Mac OS, readlink -f doesn't work.
+    file="$(readlink "$file")"
+  done
+  echo "$file"
+}
+
+# Unlike $0, $BASH_SOURCE points to the absolute path of this file.
+PROG_NAME="$(follow_links "$BASH_SOURCE")"
+
+# Handle the case where dart-sdk/bin has been symlinked to.
+CUR_DIR="$(cd "${PROG_NAME%/*}" ; pwd -P)"
+
+SDK_DIR="$CUR_DIR/../../.."
+
+if [[ `uname` == 'Darwin' ]]; then
+  DART="$SDK_DIR/tools/sdks/mac/dart-sdk/bin/dart"
+  OUT_DIR="$SDK_DIR/xcodebuild"
+else
+  DART="$SDK_DIR/tools/sdks/linux/dart-sdk/bin/dart"
+  OUT_DIR="$SDK_DIR/out"
+fi
+
+exec "$DART" "${SDK_DIR}/pkg/vm/bin/gen_kernel.dart" $@

tests/language_2/language_2_kernel.status

@@ -296,9 +296,7 @@ compile_time_constant_static5_test/16: CompileTimeError # Issue 31537
 compile_time_constant_static5_test/21: CompileTimeError # Issue 31537
 compile_time_constant_static5_test/23: CompileTimeError # Issue 31402 (Field declaration)
 conditional_import_string_test: CompileTimeError # KernelVM bug: Deferred loading kernel issue 28335.
-conditional_import_string_test: DartkCompileTimeError
 conditional_import_test: CompileTimeError # KernelVM bug: Deferred loading kernel issue 28335.
-conditional_import_test: DartkCompileTimeError
 conditional_method_invocation_test/12: MissingCompileTimeError
 conditional_method_invocation_test/13: MissingCompileTimeError
 conditional_property_access_test/10: MissingCompileTimeError

tests/lib_2/lib_2_kernel.status

@@ -65,9 +65,7 @@ async/timer_not_available_test: RuntimeError
 async/timer_repeat_test: CompileTimeError # Issue 31402 (Invocation arguments)
 async/timer_test: CompileTimeError # Issue 31402 (Invocation arguments)
 async/zone_run_unary_test: CompileTimeError # Issue 31537
-convert/streamed_conversion_json_utf8_decode_test: DartkCompileTimeError
 convert/streamed_conversion_json_utf8_decode_test: Pass, Slow # Infrequent timeouts.
-html/*: DartkCompileTimeError
 html/*: SkipByDesign # dart:html not supported on VM.
 isolate/compile_time_error_test/01: MissingCompileTimeError
 isolate/count_test: CompileTimeError # Issue 31402 (Invocation arguments)

tools/testing/dart/command.dart

@@ -52,14 +52,14 @@ class Command {
         workingDirectory: workingDirectory);
   }
 
-  static Command kernelCompilation(
+  static Command vmKernelCompilation(
       String outputFile,
       bool neverSkipCompilation,
       List<Uri> bootstrapDependencies,
       String executable,
       List<String> arguments,
       Map<String, String> environment) {
-    return new KernelCompilationCommand._(outputFile, neverSkipCompilation,
+    return new VMKernelCompilationCommand._(outputFile, neverSkipCompilation,
         bootstrapDependencies, executable, arguments, environment);
   }
 
@@ -282,15 +282,15 @@ class CompilationCommand extends ProcessCommand {
       deepJsonCompare(_bootstrapDependencies, other._bootstrapDependencies);
 }
 
-class KernelCompilationCommand extends CompilationCommand {
-  KernelCompilationCommand._(
+class VMKernelCompilationCommand extends CompilationCommand {
+  VMKernelCompilationCommand._(
       String outputFile,
       bool neverSkipCompilation,
       List<Uri> bootstrapDependencies,
       String executable,
       List<String> arguments,
       Map<String, String> environmentOverrides)
-      : super._('dartk', outputFile, neverSkipCompilation,
+      : super._('vm_compile_to_kernel', outputFile, neverSkipCompilation,
             bootstrapDependencies, executable, arguments, environmentOverrides);
 
   int get maxNumRetries => 1;

tools/testing/dart/command_output.dart

@@ -856,8 +856,8 @@ class DevCompilerCommandOutput extends CommandOutput {
   }
 }
 
-class KernelCompilationCommandOutput extends CompilationCommandOutput {
-  KernelCompilationCommandOutput(
+class VMKernelCompilationCommandOutput extends CompilationCommandOutput {
+  VMKernelCompilationCommandOutput(
       Command command,
       int exitCode,
       bool timedOut,
@@ -879,16 +879,12 @@ class KernelCompilationCommandOutput extends CompilationCommandOutput {
     Expectation result = super.result(testCase);
     if (result.canBeOutcomeOf(Expectation.crash)) {
       return Expectation.dartkCrash;
-    } else if (result.canBeOutcomeOf(Expectation.timeout)) {
-      return Expectation.dartkTimeout;
-    } else if (result.canBeOutcomeOf(Expectation.compileTimeError)) {
-      return Expectation.dartkCompileTimeError;
     }
     return result;
   }
 
   /// If the compiler was able to produce a Kernel IR file we want to run the
-  /// result on the Dart VM. We therefore mark the [KernelCompilationCommand]
+  /// result on the Dart VM. We therefore mark the [VMKernelCompilationCommand]
   /// as successful.
   ///
   /// This ensures we test that the DartVM produces correct CompileTime errors
@@ -954,8 +950,8 @@ CommandOutput createCommandOutput(Command command, int exitCode, bool timedOut,
   } else if (command is VmCommand) {
     return new VMCommandOutput(
         command, exitCode, timedOut, stdout, stderr, time, pid);
-  } else if (command is KernelCompilationCommand) {
-    return new KernelCompilationCommandOutput(
+  } else if (command is VMKernelCompilationCommand) {
+    return new VMKernelCompilationCommandOutput(
         command, exitCode, timedOut, stdout, stderr, time, compilationSkipped);
   } else if (command is AdbPrecompilationCommand) {
     return new VMCommandOutput(

tools/testing/dart/compiler_configuration.dart

@@ -595,20 +595,18 @@ class PrecompilerCompilerConfiguration extends CompilerConfiguration {
 
   Command computeCompileToKernelCommand(String tempDir, List<String> arguments,
       Map<String, String> environmentOverrides) {
-    var buildDir = _configuration.buildDirectory;
-    String exec = Platform.executable;
+    final genKernel =
+        Platform.script.resolve('../../../pkg/vm/tool/gen_kernel').toFilePath();
+    final dillFile = tempKernelFile(tempDir);
     var args = [
-      '--packages=.packages',
-      'pkg/vm/bin/gen_kernel.dart',
-      '--platform=${buildDir}/vm_platform_strong.dill',
       '--aot',
+      '--platform=${_configuration.buildDirectory}/vm_platform_strong.dill',
       '-o',
-      tempKernelFile(tempDir),
+      dillFile,
     ];
-    args.addAll(arguments.where((name) => name.endsWith('.dart')));
-    return Command.compilation('compile_to_kernel', tempDir,
-        bootstrapDependencies(), exec, args, environmentOverrides,
-        alwaysCompile: !_useSdk);
+    args.add(arguments.where((name) => name.endsWith('.dart')).single);
+    return Command.vmKernelCompilation(dillFile, true, bootstrapDependencies(),
+        genKernel, args, environmentOverrides);
   }
 
   /// Creates a command to clean up large temporary kernel files.

tools/testing/dart/test_runner.dart

@@ -1145,10 +1145,10 @@ class CommandExecutorImpl implements CommandExecutor {
   Future<CommandOutput> _runCommand(Command command, int timeout) {
     if (command is BrowserTestCommand) {
       return _startBrowserControllerTest(command, timeout);
-    } else if (command is KernelCompilationCommand) {
-      // For now, we always run dartk in batch mode.
+    } else if (command is VMKernelCompilationCommand) {
+      // For now, we always run vm_compile_to_kernel in batch mode.
       var name = command.displayName;
-      assert(name == 'dartk');
+      assert(name == 'vm_compile_to_kernel');
       return _getBatchRunner(name)
           .runCommand(name, command, timeout, command.arguments);
     } else if (command is CompilationCommand &&
@@ -1331,7 +1331,7 @@ bool shouldRetryCommand(CommandOutput output) {
 
     // The dartk batch compiler sometimes runs out of memory. In such a case we
     // will retry running it.
-    if (command is KernelCompilationCommand) {
+    if (command is VMKernelCompilationCommand) {
       if (output.hasCrashed) {
         bool containsOutOfMemoryMessage(String line) {
           return line.contains('Exhausted heap space, trying to allocat');