Merge pull request #440 from dart-lang/transformer3

Simple transformer to run ddc in pub build/serve

Author: ochafik

pkg/dev_compiler/lib/src/codegen/js_codegen.dart

@@ -3589,7 +3589,8 @@ class JSGenerator extends CodeGenerator {
         ? Uri.parse('http://${flags.host}:${flags.port}/')
         : null;
     return writeJsLibrary(module, out, compiler.inputBaseDir, serverUri,
-        emitSourceMaps: options.emitSourceMaps);
+        emitSourceMaps: options.emitSourceMaps,
+        fileSystem: compiler.fileSystem);
   }
 }
 

pkg/dev_compiler/lib/src/codegen/js_printer.dart

@@ -12,16 +12,15 @@ import 'package:source_maps/source_maps.dart' show SourceMapSpan;
 import 'package:source_span/source_span.dart' show SourceLocation;
 
 import '../js/js_ast.dart' as JS;
-import '../utils.dart' show computeHash, locationForOffset;
+import '../utils.dart' show FileSystem, computeHash, locationForOffset;
 
 import 'js_names.dart' show TemporaryNamer;
 
 String writeJsLibrary(
     JS.Program jsTree, String outputPath, String inputDir, Uri serverUri,
-    {bool emitSourceMaps: false}) {
+    {bool emitSourceMaps: false, FileSystem fileSystem}) {
   var outFilename = path.basename(outputPath);
   var outDir = path.dirname(outputPath);
-  new Directory(outDir).createSync(recursive: true);
 
   JS.JavaScriptPrintingContext context;
   if (emitSourceMaps) {
@@ -54,11 +53,11 @@ String writeJsLibrary(
     //   "names": ["state","print"]
     sourceMapText =
         sourceMapText.replaceAll('\n    ', '').replaceAll('\n  ]', ']');
-    new File('$outputPath.map').writeAsStringSync('$sourceMapText\n');
+    fileSystem.writeAsStringSync('$outputPath.map', '$sourceMapText\n');
   } else {
     text = (context as JS.SimpleJavaScriptPrintingContext).getText();
   }
-  new File(outputPath).writeAsStringSync(text);
+  fileSystem.writeAsStringSync(outputPath, text);
   if (jsTree.scriptTag != null) {
     // Mark executable.
     // TODO(jmesserly): should only do this if the input file was executable?

pkg/dev_compiler/lib/src/compiler.dart

@@ -32,7 +32,7 @@ import 'info.dart'
 import 'options.dart';
 import 'report.dart';
 import 'report/html_reporter.dart';
-import 'utils.dart' show isStrongModeError;
+import 'utils.dart' show FileSystem, isStrongModeError;
 
 /// Sets up the type checker logger to print a span that highlights error
 /// messages.
@@ -100,12 +100,13 @@ class BatchCompiler extends AbstractCompiler {
   final _pendingLibraries = <LibraryUnit>[];
 
   BatchCompiler(AnalysisContext context, CompilerOptions options,
-      {AnalysisErrorListener reporter})
+      {AnalysisErrorListener reporter,
+      FileSystem fileSystem: const FileSystem()})
       : super(
             context,
             options,
-            new ErrorCollector(
-                reporter ?? AnalysisErrorListener.NULL_LISTENER)) {
+            new ErrorCollector(reporter ?? AnalysisErrorListener.NULL_LISTENER),
+            fileSystem) {
     _inputBaseDir = options.inputBaseDir;
     if (outputDir != null) {
       _jsGen = new JSGenerator(this);
@@ -264,8 +265,7 @@ class BatchCompiler extends AbstractCompiler {
           output.lastModifiedSync() == input.lastModifiedSync()) {
         continue;
       }
-      new Directory(path.dirname(output.path)).createSync(recursive: true);
-      input.copySync(output.path);
+      fileSystem.copySync(input.path, output.path);
     }
   }
 
@@ -304,10 +304,8 @@ class BatchCompiler extends AbstractCompiler {
       }
     }
 
-    var outputFile = getOutputPath(source.uri);
-    new File(outputFile)
-      ..createSync(recursive: true)
-      ..writeAsStringSync(document.outerHtml + '\n');
+    fileSystem.writeAsStringSync(
+        getOutputPath(source.uri), document.outerHtml + '\n');
   }
 
   html.DocumentFragment _linkLibraries(
@@ -357,8 +355,10 @@ abstract class AbstractCompiler {
   final CompilerOptions options;
   final AnalysisContext context;
   final AnalysisErrorListener reporter;
+  final FileSystem fileSystem;
 
-  AbstractCompiler(this.context, this.options, [AnalysisErrorListener listener])
+  AbstractCompiler(this.context, this.options,
+      [AnalysisErrorListener listener, this.fileSystem = const FileSystem()])
       : reporter = listener ?? AnalysisErrorListener.NULL_LISTENER;
 
   String get outputDir => options.codegenOptions.outputDir;

pkg/dev_compiler/lib/src/transformer/asset_source.dart

@@ -0,0 +1,65 @@
+// Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'package:analyzer/src/generated/engine.dart';
+import 'package:analyzer/src/generated/source.dart' show Source, UriKind;
+import 'package:barback/barback.dart' show Asset, TimestampedData;
+import 'package:path/path.dart' as path;
+
+class AssetSource implements Source {
+  final Uri uri;
+  final Asset asset;
+  final String contentString;
+  AssetSource(this.uri, this.asset, this.contentString);
+
+  @override toString() => 'AssetSource($uri, ${asset.id})';
+
+  @override
+  TimestampedData<String> get contents =>
+      new TimestampedData(modificationStamp, contentString);
+
+  @override
+  String get encoding => null;
+
+  @override
+  bool exists() => true;
+
+  @override
+  String get fullName => uri.toString();
+
+  @override
+  bool get isInSystemLibrary => uriKind == UriKind.DART_URI;
+
+  @override
+  int get modificationStamp => 0;
+
+  @override
+  Uri resolveRelativeUri(Uri relativeUri) {
+    var resolvedPath = path.join(path.dirname(uri.path), relativeUri.path);
+    return new Uri(scheme: uri.scheme, path: resolvedPath);
+  }
+
+  @override
+  String get shortName => uri.toString();
+
+  @override
+  Source get source => this;
+
+  @override
+  UriKind get uriKind {
+    switch (uri.scheme) {
+      case 'package':
+        return UriKind.PACKAGE_URI;
+
+      case 'dart':
+        return UriKind.DART_URI;
+
+      case 'file':
+        return UriKind.FILE_URI;
+
+      default:
+        throw new StateError(uri.toString());
+    }
+  }
+}

pkg/dev_compiler/lib/src/transformer/asset_universe.dart

@@ -0,0 +1,52 @@
+// Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+library dev_compiler.src.transformer.asset_universe;
+
+import 'dart:async';
+
+import 'package:analyzer/analyzer.dart' show UriBasedDirective, parseDirectives;
+import 'package:barback/barback.dart' show Asset, AssetId;
+
+import 'asset_source.dart';
+import 'uri_resolver.dart' show assetIdToUri, resolveAssetId;
+
+/// Set of assets sources available for analysis / compilation.
+class AssetUniverse {
+  final _assetCache = <AssetId, AssetSource>{};
+
+  Iterable<AssetId> get assetIds => _assetCache.keys;
+
+  AssetSource getAssetSource(AssetId id) {
+    var source = _assetCache[id];
+    if (source == null) {
+      throw new ArgumentError(id.toString());
+    }
+    return source;
+  }
+
+  /// Recursively loads the asset with [id] and all its transitive dependencies.
+  Future scanSources(AssetId id, Future<Asset> getInput(AssetId id)) async {
+    if (_assetCache.containsKey(id)) return;
+
+    var asset = await getInput(id);
+    var contents = await asset.readAsString();
+    _assetCache[id] =
+        new AssetSource(Uri.parse(assetIdToUri(id)), asset, contents);
+
+    var deps = _getDependentAssetIds(id, contents);
+    await Future.wait(deps.map((depId) => scanSources(depId, getInput)));
+  }
+
+  Iterable<AssetId> _getDependentAssetIds(AssetId id, String contents) sync* {
+    var directives = parseDirectives(contents, suppressErrors: true).directives;
+    for (var directive in directives) {
+      if (directive is UriBasedDirective) {
+        var uri = directive.uri.stringValue;
+        var assetId = resolveAssetId(Uri.parse(uri), fromAssetId: id);
+        if (assetId != null) yield assetId;
+      }
+    }
+  }
+}

pkg/dev_compiler/lib/src/transformer/error_listener.dart

@@ -0,0 +1,30 @@
+// Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+library dev_compiler.src.transformer.error_listener;
+
+import 'package:barback/barback.dart' show TransformLogger;
+import 'package:analyzer/analyzer.dart'
+    show AnalysisError, ErrorSeverity, AnalysisErrorListener;
+
+class TransformAnalysisErrorListener extends AnalysisErrorListener {
+  TransformLogger _logger;
+  TransformAnalysisErrorListener(this._logger);
+
+  @override
+  void onError(AnalysisError error) {
+    // TODO(ochafik): Proper location / span.
+    switch (error.errorCode.errorSeverity) {
+      case ErrorSeverity.ERROR:
+        _logger.error(error.message);
+        break;
+      case ErrorSeverity.WARNING:
+        _logger.warning(error.message);
+        break;
+      default:
+        _logger.info(error.message);
+        break;
+    }
+  }
+}

pkg/dev_compiler/lib/src/transformer/transformer.dart

@@ -0,0 +1,134 @@
+// Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+library dev_compiler.src.transformer.transformer;
+
+import 'dart:async';
+import 'dart:io';
+
+import 'package:barback/barback.dart';
+import 'package:analyzer/src/generated/engine.dart'
+    show AnalysisEngine, AnalysisOptionsImpl;
+import 'package:path/path.dart' as path;
+
+import 'asset_universe.dart';
+import 'error_listener.dart';
+import 'uri_resolver.dart' show assetIdToUri, createSourceFactory;
+import '../compiler.dart';
+import '../options.dart';
+import '../utils.dart';
+import 'package:analyzer/src/generated/engine.dart';
+
+const String _fakeRuntimeDir = "<runtime>";
+
+/// Disclaimer: this transformer is experimental and not optimized. It may run
+/// out of memory for large applications: please use DDC's command-line runner
+/// instead whenever possible.
+class DdcTransformer extends AggregateTransformer {
+  final List<String> _ddcArgs;
+
+  DdcTransformer.asPlugin(BarbackSettings settings)
+      : _ddcArgs = settings.configuration['args'] ?? [];
+
+  @override
+  apply(AggregateTransform transform) async {
+    var inputs = await transform.primaryInputs.toList();
+
+    // The analyzer's source factory mechanism is synchronous, so we can't
+    // have it wait upon transform.getInput. Instead, we build the whole
+    // universe (scanning transitive dependencies and reading their sources),
+    // so as to get a non-async source getter.
+    // Note: This means we use a lot of memory: one way to fix it would be to
+    // propagate asynchonous calls throughout the analyzer.
+    var universe = new AssetUniverse();
+    await Future.wait(
+        inputs.map((a) => universe.scanSources(a.id, transform.getInput)));
+
+    // TODO(ochafik): invesigate the us of createAnalysisContextWithSources
+    // instead.
+    var context = AnalysisEngine.instance.createAnalysisContext();
+    context.analysisOptions = _makeAnalysisOptions();
+    context.sourceFactory = createSourceFactory(universe.getAssetSource);
+
+    // Virtual file system that writes into the transformer's outputs.
+    var fileSystem = new _TransformerFileSystem(
+        transform.logger, transform.package, transform.addOutput,
+        // Seed the runtime files into our file system:
+        inputs: await _readRuntimeFiles(transform.getInput));
+
+    var compiler = new BatchCompiler(
+        context,
+        // Note that the output directory needs not exist, and the runtime
+        // directory is a special value that corresponds to the seeding of
+        // runtimeFiles above.
+        parseOptions([]
+          ..addAll(_ddcArgs)
+          ..addAll([
+            '-o',
+            fileSystem.outputDir.path,
+            '--runtime-dir',
+            _fakeRuntimeDir
+          ])),
+        reporter: new TransformAnalysisErrorListener(transform.logger),
+        fileSystem: fileSystem);
+
+    for (var asset in inputs) {
+      compiler.compileFromUriString(assetIdToUri(asset.id));
+    }
+  }
+
+  // TODO(ochafik): Provide more control over these options.
+  AnalysisOptions _makeAnalysisOptions() => new AnalysisOptionsImpl()
+    ..cacheSize = 256 // # of sources to cache ASTs for.
+    ..preserveComments = true
+    ..analyzeFunctionBodies = true
+    ..strongMode = true;
+
+  /// Read the runtime files from the transformer (they're available as
+  /// resources of package:dev_compiler),
+  Future<Map<String, String>> _readRuntimeFiles(
+      Future<Asset> getInput(AssetId id)) async {
+    var runtimeFiles = <String, String>{};
+    for (var file in defaultRuntimeFiles) {
+      var asset =
+          await getInput(new AssetId('dev_compiler', 'lib/runtime/$file'));
+      runtimeFiles[path.join(_fakeRuntimeDir, file)] =
+          await asset.readAsString();
+    }
+    return runtimeFiles;
+  }
+
+  /// We just transform all .dart and .html files in one go.
+  @override
+  classifyPrimary(AssetId id) =>
+      id.extension == '.dart' || id.extension == '.html' ? '<dart>' : null;
+}
+
+/// Type of [Transform.addOutput] and [AggregateTransform.addOutput].
+typedef void AssetOutputAdder(Asset asset);
+
+/// Virtual file system that outputs files into a transformer.
+class _TransformerFileSystem implements FileSystem {
+  final String _package;
+  final Directory outputDir = Directory.current;
+  final String outputPrefix;
+  final AssetOutputAdder _addOutput;
+  final Map<String, String> inputs;
+  final TransformLogger _logger;
+  _TransformerFileSystem(this._logger, this._package, this._addOutput,
+      {this.inputs, this.outputPrefix: 'web/'});
+
+  @override
+  void writeAsStringSync(String file, String contents) {
+    var id = new AssetId(
+        _package, outputPrefix + path.relative(file, from: outputDir.path));
+    _logger.fine('Adding output $id');
+    _addOutput(new Asset.fromString(id, contents));
+  }
+
+  @override
+  void copySync(String src, String dest) {
+    writeAsStringSync(dest, inputs[src] ?? new File(src).readAsStringSync());
+  }
+}