Introduce transformer support for tsickle

API (see `transformer.ts`): The method `emitWithTsickle` allows to emit
any program with tsickle. I.e. this is a replacement for using the `TsickleCompilerHost`.

Implementation using existing `DecoratorClassVisitor` and `Annotator`:
(see `transformer_sourcemap.ts`): Parses the produced sources back
into a `ts.SourceFile` and establishes the links to the original
nodes via the information given to the `SourceMapper` interface.

Tests via goldens:
The transformer version of Tsickle emits slightly different JavaScript
(see the previous commit). Therefore, some goldens are slightly different.
We use patch files to assert the differences to the regular goldens.

Tests for SourceMaps:
TypeScript automatically produces source maps for all changes
that the transformer does. However, some applications (e.g. Angular)
rely on the feature of tsickle to merge source maps
that were already present in the input TypeScript code.
Therefore, the transformer version still has to implement this merging.

Author: tbosch

package.json

@@ -19,6 +19,7 @@
   },
   "devDependencies": {
     "@types/chai": "^3.4.32",
+    "@types/diff": "^3.2.0",
     "@types/glob": "^5.0.29",
     "@types/google-closure-compiler": "0.0.18",
     "@types/minimatch": "^2.0.28",
@@ -30,6 +31,7 @@
     "@types/source-map-support": "^0.2.27",
     "chai": "^3.5.0",
     "clang-format": "^1.0.51",
+    "diff": "^3.2.0",
     "glob": "^7.0.0",
     "google-closure-compiler": "^20161024.1.0",
     "gulp": "^3.8.11",

src/modules_manifest.ts

@@ -15,6 +15,11 @@ export class ModulesManifest {
   /** Map of file name to arrays of imported googmodule module names */
   private referencedModules: FileMap<string[]> = {};
 
+  addManifest(other: ModulesManifest) {
+    Object.assign(this.moduleToFileName, other.moduleToFileName);
+    Object.assign(this.referencedModules, other.referencedModules);
+  }
+
   addModule(fileName: string, module: string): void {
     this.moduleToFileName[module] = fileName;
     this.referencedModules[fileName] = [];

src/source_map_utils.ts

@@ -6,7 +6,7 @@
  * found in the LICENSE file at https://angular.io/license
  */
 
-import {SourceMapConsumer, SourceMapGenerator} from 'source-map';
+import {RawSourceMap, SourceMapConsumer, SourceMapGenerator} from 'source-map';
 import * as ts from 'typescript';
 
 /**
@@ -62,6 +62,17 @@ export function setInlineSourceMap(source: string, sourceMap: string): string {
   }
 }
 
+export function parseSourceMap(text: string, fileName?: string, sourceName?: string): RawSourceMap {
+  const rawSourceMap = JSON.parse(text) as RawSourceMap;
+  if (sourceName) {
+    rawSourceMap.sources = [sourceName];
+  }
+  if (fileName) {
+    rawSourceMap.file = fileName;
+  }
+  return rawSourceMap;
+}
+
 export function sourceMapConsumerToGenerator(sourceMapConsumer: SourceMapConsumer):
     SourceMapGenerator {
   return SourceMapGenerator.fromSourceMap(sourceMapConsumer);

src/transformer.ts

@@ -0,0 +1,222 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import * as path from 'path';
+import {RawSourceMap, SourceMapConsumer, SourceMapGenerator} from 'source-map';
+import * as ts from 'typescript';
+
+import * as decorator from './decorator-annotator';
+import * as es5processor from './es5processor';
+import {ModulesManifest} from './modules_manifest';
+import {containsInlineSourceMap, extractInlineSourceMap, parseSourceMap, removeInlineSourceMap, setInlineSourceMap, SourceMapper, SourcePosition} from './source_map_utils';
+import {createTransformerFromSourceMap} from './transformer_sourcemap';
+import {createCustomTransformers} from './transformer_util';
+import * as tsickle from './tsickle';
+
+export interface TransformerOptions extends es5processor.Es5ProcessorOptions, tsickle.Options {
+  /**
+   * Whether to downlevel decorators
+   */
+  transformDecorators?: boolean;
+  /**
+   * Whether to convers types to closure
+   */
+  transformTypesToClosure?: boolean;
+}
+
+export interface TransformerHost extends es5processor.Es5ProcessorHost, tsickle.AnnotatorHost {
+  /**
+   * If true, tsickle and decorator downlevel processing will be skipped for
+   * that file.
+   */
+  shouldSkipTsickleProcessing(fileName: string): boolean;
+  /**
+   * Tsickle treats warnings as errors, if true, ignore warnings.  This might be
+   * useful for e.g. third party code.
+   */
+  shouldIgnoreWarningsForPath(filePath: string): boolean;
+}
+
+export function mergeEmitResults(emitResults: EmitResult[]): EmitResult {
+  const diagnostics: ts.Diagnostic[] = [];
+  let emitSkipped = true;
+  const emittedFiles: string[] = [];
+  const externs: {[fileName: string]: string} = {};
+  const modulesManifest = new ModulesManifest();
+  emitResults.forEach(er => {
+    diagnostics.push(...er.diagnostics);
+    emitSkipped = emitSkipped || er.emitSkipped;
+    emittedFiles.push(...er.emittedFiles);
+    Object.assign(externs, er.externs);
+    modulesManifest.addManifest(er.modulesManifest);
+  });
+  return {diagnostics, emitSkipped, emittedFiles, externs, modulesManifest};
+}
+
+export interface EmitResult extends ts.EmitResult {
+  // The manifest of JS modules output by the compiler.
+  modulesManifest: ModulesManifest;
+  /** externs.js files produced by tsickle, if any. */
+  externs: {[fileName: string]: string};
+}
+
+export interface EmitTransformers {
+  beforeTsickle?: Array<ts.TransformerFactory<ts.SourceFile>>;
+  beforeTs?: Array<ts.TransformerFactory<ts.SourceFile>>;
+  afterTs?: Array<ts.TransformerFactory<ts.SourceFile>>;
+}
+
+export function emitWithTsickle(
+    program: ts.Program, host: TransformerHost, options: TransformerOptions,
+    tsHost: ts.CompilerHost, tsOptions: ts.CompilerOptions, targetSourceFile?: ts.SourceFile,
+    writeFile?: ts.WriteFileCallback, cancellationToken?: ts.CancellationToken,
+    emitOnlyDtsFiles?: boolean, customTransformers?: EmitTransformers): EmitResult {
+  let tsickleDiagnostics: ts.Diagnostic[] = [];
+  const typeChecker = program.getTypeChecker();
+  const beforeTsTransformers: Array<ts.TransformerFactory<ts.SourceFile>> = [];
+  // add tsickle transformers
+  if (options.transformTypesToClosure) {
+    // Note: tsickle.annotate can also lower decorators in the same run.
+    beforeTsTransformers.push(createTransformerFromSourceMap((sourceFile, sourceMapper) => {
+      const tisckleOptions: tsickle.Options = {...options, filterTypesForExport: true};
+      const {output, diagnostics} = tsickle.annotate(
+          typeChecker, sourceFile, host, tisckleOptions, tsHost, tsOptions, sourceMapper,
+          tsickle.AnnotatorFeatures.Transformer);
+      tsickleDiagnostics.push(...diagnostics);
+      return output;
+    }));
+  } else if (options.transformDecorators) {
+    beforeTsTransformers.push(createTransformerFromSourceMap((sourceFile, sourceMapper) => {
+      const {output, diagnostics} =
+          decorator.convertDecorators(typeChecker, sourceFile, sourceMapper);
+      tsickleDiagnostics.push(...diagnostics);
+      return output;
+    }));
+  }
+  // // For debugging: transformer that just emits the same text.
+  // beforeTsTransformers.push(createTransformer(host, typeChecker, (sourceFile, sourceMapper) => {
+  //   sourceMapper.addMapping(sourceFile, {position: 0, line: 0, column: 0}, {position: 0, line: 0,
+  //   column: 0}, sourceFile.text.length); return sourceFile.text;
+  // }));
+  // add user supplied transformers
+  const afterTsTransformers: Array<ts.TransformerFactory<ts.SourceFile>> = [];
+  if (customTransformers) {
+    if (customTransformers.beforeTsickle) {
+      beforeTsTransformers.unshift(...customTransformers.beforeTsickle);
+    }
+
+    if (customTransformers.beforeTs) {
+      beforeTsTransformers.push(...customTransformers.beforeTs);
+    }
+    if (customTransformers.afterTs) {
+      afterTsTransformers.push(...customTransformers.afterTs);
+    }
+  }
+  customTransformers = createCustomTransformers({
+    before: beforeTsTransformers.map(tf => skipTransformForSourceFileIfNeeded(host, tf)),
+    after: afterTsTransformers.map(tf => skipTransformForSourceFileIfNeeded(host, tf))
+  });
+
+  const writeFileDelegate = writeFile || tsHost.writeFile.bind(tsHost);
+  const modulesManifest = new ModulesManifest();
+  const writeFileImpl =
+      (fileName: string, content: string, writeByteOrderMark: boolean,
+       onError?: (message: string) => void, sourceFiles?: ts.SourceFile[]) => {
+        if (path.extname(fileName) !== '.map') {
+          if (tsOptions.inlineSourceMap) {
+            content = combineInlineSourceMaps(program, fileName, content);
+          } else {
+            content = removeInlineSourceMap(content);
+          }
+          content = es5processor.convertCommonJsToGoogModuleIfNeeded(
+              host, options, modulesManifest, fileName, content);
+        } else {
+          content = combineSourceMaps(program, fileName, content);
+        }
+        writeFileDelegate(fileName, content, writeByteOrderMark, onError, sourceFiles);
+      };
+
+  const {diagnostics: tsDiagnostics, emitSkipped, emittedFiles} = program.emit(
+      targetSourceFile, writeFileImpl, cancellationToken, emitOnlyDtsFiles, customTransformers);
+
+  const externs: {[fileName: string]: string} = {};
+  if (options.transformTypesToClosure) {
+    const sourceFiles = targetSourceFile ? [targetSourceFile] : program.getSourceFiles();
+    sourceFiles.forEach(sf => {
+      if (tsickle.isDtsFileName(sf.fileName) && host.shouldSkipTsickleProcessing(sf.fileName)) {
+        return;
+      }
+      const {output, diagnostics} = tsickle.writeExterns(typeChecker, sf, host, options);
+      if (output) {
+        externs[sf.fileName] = output;
+      }
+      if (diagnostics) {
+        tsickleDiagnostics.push(...diagnostics);
+      }
+    });
+  }
+  // All diagnostics (including warnings) are treated as errors.
+  // If we've decided to ignore them, just discard them.
+  // Warnings include stuff like "don't use @type in your jsdoc"; tsickle
+  // warns and then fixes up the code to be Closure-compatible anyway.
+  tsickleDiagnostics = tsickleDiagnostics.filter(
+      d => d.category === ts.DiagnosticCategory.Error ||
+          !host.shouldIgnoreWarningsForPath(d.file.fileName));
+
+  return {
+    modulesManifest,
+    emitSkipped,
+    emittedFiles: emittedFiles || [],
+    diagnostics: [...tsDiagnostics, ...tsickleDiagnostics],
+    externs
+  };
+}
+
+function skipTransformForSourceFileIfNeeded(
+    host: TransformerHost,
+    delegateFactory: ts.TransformerFactory<ts.SourceFile>): ts.TransformerFactory<ts.SourceFile> {
+  return (context: ts.TransformationContext) => {
+    const delegate = delegateFactory(context);
+    return (sourceFile: ts.SourceFile) => {
+      if (host.shouldSkipTsickleProcessing(sourceFile.fileName)) {
+        return sourceFile;
+      }
+      return delegate(sourceFile);
+    };
+  };
+}
+
+function combineInlineSourceMaps(
+    program: ts.Program, filePath: string, compiledJsWithInlineSourceMap: string): string {
+  if (tsickle.isDtsFileName(filePath)) {
+    return compiledJsWithInlineSourceMap;
+  }
+  const sourceMapJson = extractInlineSourceMap(compiledJsWithInlineSourceMap);
+  compiledJsWithInlineSourceMap = removeInlineSourceMap(compiledJsWithInlineSourceMap);
+  const composedSourceMap = combineSourceMaps(program, filePath, sourceMapJson);
+  return setInlineSourceMap(compiledJsWithInlineSourceMap, composedSourceMap);
+}
+
+function combineSourceMaps(
+    program: ts.Program, filePath: string, tscSourceMapText: string): string {
+  const tscSourceMap = parseSourceMap(tscSourceMapText);
+  let tscSourceMapGenerator: SourceMapGenerator|undefined;
+  for (const sourceFileName of tscSourceMap.sources) {
+    const sourceFile = program.getSourceFile(sourceFileName);
+    if (!sourceFile || !containsInlineSourceMap(sourceFile.text)) {
+      continue;
+    }
+    const preexistingSourceMapText = extractInlineSourceMap(sourceFile.text);
+    if (!tscSourceMapGenerator) {
+      tscSourceMapGenerator = SourceMapGenerator.fromSourceMap(new SourceMapConsumer(tscSourceMap));
+    }
+    tscSourceMapGenerator.applySourceMap(
+        new SourceMapConsumer(parseSourceMap(preexistingSourceMapText, sourceFileName)));
+  }
+  return tscSourceMapGenerator ? tscSourceMapGenerator.toString() : tscSourceMapText;
+}