diff --git a/.travis.yml b/.travis.yml
index d9813c298474..73e3c4d84d12 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -12,6 +12,7 @@ matrix:
allow_failures:
- env: NODE_SCRIPT="tests/run_e2e.js --nightly"
- env: NODE_SCRIPT="tests/run_e2e.js --ng2"
+ - env: NODE_SCRIPT="tests/run_e2e.js --glob=benchmarks/**"
- node_js: "7"
include:
- node_js: "6"
@@ -40,6 +41,9 @@ matrix:
- node_js: "6"
os: linux
env: NODE_SCRIPT="tests/run_e2e.js --nightly"
+ - node_js: "6"
+ os: linux
+ env: NODE_SCRIPT="tests/run_e2e.js --glob=benchmarks/**"
- node_js: "7"
os: linux
env: NODE_SCRIPT=tests/run_e2e.js
diff --git a/bin/ngbench b/bin/ngbench
new file mode 100644
index 000000000000..dace66b43c76
--- /dev/null
+++ b/bin/ngbench
@@ -0,0 +1,8 @@
+#!/usr/bin/env node
+'use strict';
+
+// Provide a title to the process in `ps`
+process.title = 'bench';
+
+require('../lib/bootstrap-local');
+require('../tools/bench/bin/ngbench');
diff --git a/package.json b/package.json
index 2fe90c297ba1..b9ab0ba39727 100644
--- a/package.json
+++ b/package.json
@@ -5,7 +5,8 @@
"main": "packages/@angular/cli/lib/cli/index.js",
"trackingCode": "UA-8594346-19",
"bin": {
- "ng": "./bin/ng"
+ "ng": "./bin/ng",
+ "ngbench": "./bin/ngbench"
},
"keywords": [],
"scripts": {
@@ -126,6 +127,7 @@
"@types/semver": "^5.3.30",
"@types/source-map": "^0.5.0",
"@types/webpack": "^2.2.15",
+ "@types/yargs": "^6.5.0",
"chai": "^3.5.0",
"conventional-changelog": "^1.1.0",
"dtsgenerator": "^0.9.1",
@@ -145,10 +147,12 @@
"rewire": "^2.5.1",
"sinon": "^1.17.3",
"spdx-satisfies": "^0.1.3",
+ "strip-ansi": "^3.0.1",
"through": "^2.3.6",
- "tree-kill": "^1.0.0",
+ "tree-kill": "^1.1.0",
"ts-node": "^2.0.0",
- "tslint": "^5.1.0"
+ "tslint": "^5.1.0",
+ "yargs": "^6.6.0"
},
"optionalDependencies": {
"node-sass": "^4.3.0"
diff --git a/tests/e2e/benchmarks/build.ts b/tests/e2e/benchmarks/build.ts
new file mode 100644
index 000000000000..3398498408d1
--- /dev/null
+++ b/tests/e2e/benchmarks/build.ts
@@ -0,0 +1,6 @@
+import { ngbench } from '../utils/process';
+
+
+export default function () {
+ return ngbench('--command', 'ng build');
+}
diff --git a/tests/e2e/benchmarks/serve/css.ts b/tests/e2e/benchmarks/serve/css.ts
new file mode 100644
index 000000000000..eb57a66efb40
--- /dev/null
+++ b/tests/e2e/benchmarks/serve/css.ts
@@ -0,0 +1,42 @@
+import { ngbench } from '../../utils/process';
+import { replaceInFile, moveFile } from '../../utils/fs';
+import { updateJsonFile } from '../../utils/project';
+
+
+export default function () {
+ const extensions = ['css', 'scss', 'less', 'styl'];
+ let promise = Promise.resolve();
+
+ extensions.forEach(ext => {
+ promise = promise.then(() => {
+ // change files to use preprocessor
+ return updateJsonFile('.angular-cli.json', configJson => {
+ const app = configJson['apps'][0];
+ app['styles'] = [`styles.${ext}`];
+ })
+ .then(() => replaceInFile('src/app/app.component.ts',
+ './app.component.css', `./app.component.${ext}`))
+ .then(() => moveFile('src/styles.css', `src/styles.${ext}`))
+ .then(() => moveFile('src/app/app.component.css', `src/app/app.component.${ext}`))
+ // run benchmarks
+ .then(() => ngbench(
+ '--comment', 'component styles',
+ '--match-edit-file', `src/app/app.component.${ext}`,
+ '--match-edit-string', 'h1{color:blue}'
+ ))
+ .then(() => ngbench(
+ '--comment', `.${ext} CSS extension`,
+ '--match-edit-file', `src/styles.${ext}`,
+ '--match-edit-string', 'h1{color:blue}'
+ ))
+ // change files back
+ .then(() => replaceInFile('src/app/app.component.ts',
+ `./app.component.${ext}`, './app.component.css'))
+ .then(() => moveFile(`src/styles.${ext}`, 'src/styles.css'))
+ .then(() => moveFile(`src/app/app.component.${ext}`, 'src/app/app.component.css'));
+
+ });
+ });
+
+ return promise;
+}
diff --git a/tests/e2e/benchmarks/serve/html.ts b/tests/e2e/benchmarks/serve/html.ts
new file mode 100644
index 000000000000..352e40b4fa2c
--- /dev/null
+++ b/tests/e2e/benchmarks/serve/html.ts
@@ -0,0 +1,9 @@
+import { ngbench } from '../../utils/process';
+
+
+export default function () {
+ return ngbench(
+ '--match-edit-file', 'src/app/app.component.html',
+ '--match-edit-string', '
'
+ );
+}
diff --git a/tests/e2e/benchmarks/serve/scripts.ts b/tests/e2e/benchmarks/serve/scripts.ts
new file mode 100644
index 000000000000..11aee8b6f5e6
--- /dev/null
+++ b/tests/e2e/benchmarks/serve/scripts.ts
@@ -0,0 +1,16 @@
+import { ngbench } from '../../utils/process';
+import { writeFile } from '../../utils/fs';
+import { updateJsonFile } from '../../utils/project';
+
+
+export default function () {
+ return updateJsonFile('.angular-cli.json', configJson => {
+ const app = configJson['apps'][0];
+ app['scripts'] = ['scripts.js'];
+ })
+ .then(() => writeFile('src/scripts.js', ''))
+ .then(() => ngbench(
+ '--match-edit-file', 'src/scripts.js',
+ '--match-edit-string', 'console.log(1);'
+ ));
+}
diff --git a/tests/e2e/benchmarks/serve/serve.ts b/tests/e2e/benchmarks/serve/serve.ts
new file mode 100644
index 000000000000..cfc92583f61d
--- /dev/null
+++ b/tests/e2e/benchmarks/serve/serve.ts
@@ -0,0 +1,6 @@
+import { ngbench } from '../../utils/process';
+
+
+export default function () {
+ return ngbench('--extra-args=--aot');
+}
diff --git a/tests/e2e/benchmarks/version.ts b/tests/e2e/benchmarks/version.ts
new file mode 100644
index 000000000000..e429291699c9
--- /dev/null
+++ b/tests/e2e/benchmarks/version.ts
@@ -0,0 +1,6 @@
+import { ngbench } from '../utils/process';
+
+
+export default function () {
+ return ngbench('--command', 'ng version');
+}
diff --git a/tests/e2e/utils/process.ts b/tests/e2e/utils/process.ts
index 32ff695efaac..5ca0d0f65782 100644
--- a/tests/e2e/utils/process.ts
+++ b/tests/e2e/utils/process.ts
@@ -1,5 +1,6 @@
import * as child_process from 'child_process';
import {blue, yellow} from 'chalk';
+import {join} from 'path';
import {getGlobalVariable} from './env';
import {rimraf, writeFile} from './fs';
const treeKill = require('tree-kill');
@@ -32,7 +33,7 @@ function _exec(options: ExecOptions, cmd: string, args: string[]): Promise !!x) // Remove false and undefined.
+ .filter(x => !!x) // Remove false and undefined.
.join(', ')
.replace(/^(.+)$/, ' [$1]'); // Proper formatting.
@@ -185,3 +186,8 @@ export function git(...args: string[]) {
export function silentGit(...args: string[]) {
return _exec({silent: true}, 'git', args);
}
+
+export function ngbench(...args: string[]) {
+ const ngbenchJs = join(__dirname, '../../../bin/ngbench');
+ return _exec({}, 'node', [ngbenchJs, ...args]);
+}
diff --git a/tools/bench/bin/ngbench b/tools/bench/bin/ngbench
new file mode 100644
index 000000000000..3a54e2fe01bd
--- /dev/null
+++ b/tools/bench/bin/ngbench
@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+require('../src/index');
diff --git a/tools/bench/package.json b/tools/bench/package.json
new file mode 100644
index 000000000000..d1d2ebecdfab
--- /dev/null
+++ b/tools/bench/package.json
@@ -0,0 +1,20 @@
+{
+ "name": "bench",
+ "version": "0.0.0",
+ "private": false,
+ "description": "",
+ "main": "./src/index.js",
+ "bin": {
+ "ngbench": "./bin/ngbench"
+ },
+ "license": "MIT",
+ "dependencies": {
+ "@ngtools/logger": "^0.1.4",
+ "chalk": "^1.1.3",
+ "lodash": "^4.11.1",
+ "rxjs": "^5.0.1",
+ "strip-ansi": "^3.0.1",
+ "tree-kill": "^1.1.0",
+ "yargs": "^6.6.0"
+ }
+}
diff --git a/tools/bench/src/benchmark-options.ts b/tools/bench/src/benchmark-options.ts
new file mode 100644
index 000000000000..7023f19f310f
--- /dev/null
+++ b/tools/bench/src/benchmark-options.ts
@@ -0,0 +1,12 @@
+export interface BenchmarkOptions {
+ command: string;
+ iterations: number;
+ extraArgs: string[];
+ match: string;
+ matchCount: number;
+ matchEditFile: string;
+ matchEditString: string;
+ comment: string;
+ logFile: string;
+ debug: boolean;
+}
diff --git a/tools/bench/src/benchmark.ts b/tools/bench/src/benchmark.ts
new file mode 100644
index 000000000000..3fed13263e47
--- /dev/null
+++ b/tools/bench/src/benchmark.ts
@@ -0,0 +1,114 @@
+import { Logger } from '@ngtools/logger';
+import * as fs from 'fs';
+const mean = require('lodash/mean');
+const toNumber = require('lodash/toNumber');
+
+
+import { BenchmarkOptions } from './benchmark-options';
+import {
+ combine,
+ makeMatchFn,
+ matchSpawn,
+ serialMultiPromise
+} from './utils';
+
+export function benchmark(benchmarkOptions: BenchmarkOptions, logger: Logger) {
+ const logIfDefined = (str: string, prop: any) => {
+ if (Array.isArray(prop) && prop.length === 0) {
+ prop = null;
+ }
+ return prop ? logger.info(str + prop) : null;
+ };
+
+ const cwd = process.cwd();
+
+ // Build match function
+ let matchFn: any = null;
+ if (benchmarkOptions.match) {
+ matchFn = makeMatchFn(
+ logger,
+ benchmarkOptions.match,
+ benchmarkOptions.matchCount,
+ benchmarkOptions.matchEditFile,
+ benchmarkOptions.matchEditString,
+ );
+ }
+
+ let editedFileContents: string;
+ if (benchmarkOptions.matchEditFile) {
+ // backup contents of file that is being edited for rebuilds
+ editedFileContents = fs.readFileSync(benchmarkOptions.matchEditFile, 'utf8');
+ }
+ // combine flags commands
+ let flagCombinations = combine(benchmarkOptions.extraArgs);
+ flagCombinations.unshift([]);
+
+ const startTime = Date.now();
+ logger.info(`Base command: ${benchmarkOptions.command}`);
+ logger.info(`Iterations: ${benchmarkOptions.iterations}`);
+ logIfDefined('Comment: ', benchmarkOptions.comment);
+ logIfDefined('Extra args: ', benchmarkOptions.extraArgs);
+ logIfDefined('Logging to: ', benchmarkOptions.logFile);
+ if (benchmarkOptions.match) {
+ logger.info(`Match output: ${benchmarkOptions.match}`);
+ logIfDefined('Match count: ', benchmarkOptions.matchCount);
+ logIfDefined('Match edit file: ', benchmarkOptions.matchEditFile);
+ logIfDefined('Match edit string: ', benchmarkOptions.matchEditString);
+ }
+ if (benchmarkOptions.debug) {
+ logger.debug('### Debug mode, all output is logged ###');
+ }
+ logger.info('');
+
+
+ let promise = Promise.resolve();
+ let hasFailures = false;
+
+ flagCombinations.forEach((flags) =>
+ promise = promise
+ .then(() => serialMultiPromise(
+ benchmarkOptions.iterations,
+ matchSpawn,
+ logger,
+ cwd,
+ matchFn,
+ benchmarkOptions.command,
+ flags
+ ).then((results: any[]) => {
+ const failures = results.filter(result => result.err && result.err !== 0);
+ logger.info(`Full command: ${benchmarkOptions.command} ${flags.join(' ')}`);
+
+ let times = results.filter(result => !result.err)
+ .map((result) => result.time);
+ logger.info(`Time average: ${mean(times)}`);
+ logger.info(`Times: ${times.join()}`);
+
+ if (benchmarkOptions.match) {
+ let matches = results.filter(result => !result.err)
+ .map((result) => result.match);
+
+ let matchesAsNumbers = matches.map(toNumber);
+ if (matches.every(match => match != NaN)) {
+ logger.info(`Match average: ${mean(matchesAsNumbers)}`);
+ }
+ logger.info(`Matches: ${matches.join()}`);
+ }
+
+ if (failures.length > 0) {
+ hasFailures = true;
+ logger.info(`Failures: ${failures.length}`);
+ logger.info(JSON.stringify(failures));
+ }
+ logger.info('');
+ }))
+ );
+
+ return promise.then(() => {
+ logger.info(`Benchmark execution time: ${Date.now() - startTime}ms`);
+ // restore contents of file that was being edited for rebuilds
+ if (benchmarkOptions.matchEditFile) {
+ fs.writeFileSync(benchmarkOptions.matchEditFile, editedFileContents, 'utf8');
+ }
+ return hasFailures ? Promise.reject(new Error('Some benchmarks failed')) : Promise.resolve();
+ });
+}
diff --git a/tools/bench/src/index.ts b/tools/bench/src/index.ts
new file mode 100644
index 000000000000..ecb26fb787d9
--- /dev/null
+++ b/tools/bench/src/index.ts
@@ -0,0 +1,126 @@
+import { Logger, LogEntry } from '@ngtools/logger';
+import { bold, red, yellow, white } from 'chalk';
+import * as fs from 'fs';
+import * as path from 'path';
+import * as yargs from 'yargs';
+import 'rxjs/add/operator/filter';
+
+import { BenchmarkOptions } from './benchmark-options';
+import { benchmark } from './benchmark';
+
+
+// Set options via yargs
+const benchmarkOptions: BenchmarkOptions = yargs
+ .usage('$0 [args]')
+ .options({
+ 'command': {
+ description: 'Command to benchmark, defaults to benchmarking `ng serve` if not set',
+ type: 'string',
+ alias: 'c'
+ },
+ 'iterations': {
+ description: 'Number of iterations to run benchmark',
+ type: 'number',
+ default: 5,
+ alias: 'i'
+ },
+ 'extra-args': {
+ description: 'Extra arguments to combine and run',
+ type: 'array',
+ default: [],
+ alias: 'ea'
+ },
+ 'match': {
+ description: 'Command output to match',
+ type: 'string',
+ alias: 'm'
+ },
+ 'match-count': {
+ description: 'Times to match output',
+ type: 'number',
+ alias: 'mc'
+ },
+ 'match-edit-file': {
+ description: 'File to edit after a output match',
+ type: 'string',
+ alias: 'mef'
+ },
+ 'match-edit-string': {
+ description: 'String to use in --match-edit-file',
+ type: 'string',
+ alias: 'mes'
+ },
+ 'comment': {
+ description: 'Comment to add to output',
+ type: 'string',
+ alias: 'cm'
+ },
+ 'log-file': {
+ description: 'File to log output',
+ type: 'string',
+ alias: 'lf'
+ },
+ 'debug': {
+ description: 'Show command output',
+ type: 'boolean',
+ default: false,
+ alias: 'd'
+ },
+ })
+ .help()
+ .argv;
+
+// Initialize logger
+const logger = new Logger('ngbench');
+
+logger
+ .filter((entry: LogEntry) => (entry.level != 'debug' || benchmarkOptions.debug))
+ .subscribe((entry: LogEntry) => {
+ let color: (s: string) => string = white;
+ let output = process.stdout;
+ switch (entry.level) {
+ case 'info': color = white; break;
+ case 'warn': color = yellow; break;
+ case 'error': color = red; output = process.stderr; break;
+ case 'fatal': color = (x: string) => bold(red(x)); output = process.stderr; break;
+ }
+
+ output.write(color(entry.message) + '\n');
+ if (benchmarkOptions.logFile) {
+ fs.appendFileSync(benchmarkOptions.logFile, entry.message + '\n');
+ }
+ });
+
+logger
+ .filter((entry: LogEntry) => entry.level == 'fatal')
+ .subscribe(() => {
+ process.stderr.write('A fatal error happened. See details above.');
+ process.exit(1);
+ });
+
+
+// Set compound defauls and resolve paths
+if (!benchmarkOptions.command) {
+ benchmarkOptions.command = 'ng serve --no-progress';
+ benchmarkOptions.match = benchmarkOptions.match || 'Time: (.*)ms';
+ benchmarkOptions.matchCount = benchmarkOptions.matchCount || 4;
+ benchmarkOptions.matchEditFile = benchmarkOptions.matchEditFile || 'src/main.ts';
+ benchmarkOptions.matchEditString = benchmarkOptions.matchEditString || 'console.log(1);';
+}
+
+if (benchmarkOptions.matchEditFile) {
+ benchmarkOptions.matchEditFile = path.resolve('./', benchmarkOptions.matchEditFile);
+}
+
+if (benchmarkOptions.command.match(/^ng (build|serve)/)
+ && benchmarkOptions.command.match(/--no-progress/) === null
+) {
+ logger.warn('Auto-added \'--no-progress\' to build/serve command.');
+ benchmarkOptions.command = benchmarkOptions.command + ' --no-progress';
+}
+
+// Run benchmark
+benchmark(benchmarkOptions, logger)
+ .catch((err) => {
+ logger.fatal(JSON.stringify(err));
+ });
diff --git a/tools/bench/src/utils.ts b/tools/bench/src/utils.ts
new file mode 100644
index 000000000000..a0ced018e812
--- /dev/null
+++ b/tools/bench/src/utils.ts
@@ -0,0 +1,138 @@
+import * as fs from 'fs';
+import * as child_process from 'child_process';
+import { Logger } from '@ngtools/logger';
+const stripAnsi = require('strip-ansi');
+const treeKill = require('tree-kill');
+
+// Make all combinations of array elements
+// http://stackoverflow.com/a/5752056/2116927
+export function combine(array: any[]) {
+ let fn = function (n: number, src: any[], got: any[], all: any[]) {
+ if (n == 0) {
+ if (got.length > 0) {
+ all[all.length] = got;
+ }
+ return;
+ }
+ for (let j = 0; j < src.length; j++) {
+ fn(n - 1, src.slice(j + 1), got.concat([src[j]]), all);
+ }
+ return;
+ };
+ let all: any[] = [];
+ for (let i = 0; i < array.length; i++) {
+ fn(i, array, [], all);
+ }
+ if (array.length > 0) { all.push(array); }
+ return all;
+}
+
+// Returns a promise with the result of calling `times` times the `fn` function with `args`
+// `fn` will be called in serial, and is expected to return a promise
+export function serialMultiPromise(times: number, fn: any, ...args: any[]) {
+ let results: Promise[] = [];
+ let promise = Promise.resolve();
+ Array.from({ length: times }).forEach(() => promise = promise.then(() =>
+ fn.apply(null, args).then((result: Promise) => results.push(result))
+ ));
+ return promise.then(() => results);
+}
+
+// Spawns `cmd` with `args`
+// Calls `matchFn` with the `stdout` output, a `result` var and the process
+// `dataFn` is expected to modify `result`
+export function matchSpawn(logger: Logger, cwd: string, matchFn: any,
+ cmd: string, args: string[] = []) {
+ // dataFn will have access to result and use it to store results
+ let result = {
+ // overrideErr will signal that an error code on the exit event should be ignored
+ // This is useful on windows where killing a tree of processes always makes them
+ // exit with an error code
+ overrideErr: false,
+ time: 0
+ };
+ let stdout = '';
+ let stderr = '';
+ let spawnOptions: any = {
+ cwd: cwd,
+ env: process.env
+ };
+
+ // Split given `cmd` and pass on any args to `args`
+ const splitCmd = cmd.split(' ');
+ cmd = splitCmd.shift();
+ args = splitCmd.concat(args);
+
+ if (process.platform.startsWith('win')) {
+ args = ['/c', cmd].concat(args);
+ cmd = 'cmd.exe';
+ spawnOptions['stdio'] = 'pipe';
+ }
+
+ logger.debug(`spawning cmd: ${cmd}, args: ${args}, cwd: ${cwd}`);
+
+ const startTime = Date.now();
+ const childProcess = child_process.spawn(cmd, args, spawnOptions);
+
+ childProcess.stdout.on('data', (data: Buffer) => {
+ let strippedData = stripAnsi(data.toString('utf-8'));
+ logger.debug(strippedData);
+ stdout += strippedData;
+ if (matchFn) {
+ matchFn(strippedData, result, childProcess);
+ }
+ });
+
+ childProcess.stderr.on('data', (data: Buffer) => {
+ let strippedData = stripAnsi(data.toString('utf-8'));
+ logger.debug(strippedData);
+ stderr += strippedData;
+ });
+
+ return new Promise((resolve, _) =>
+ childProcess.on('exit', (err: number) => {
+ result.time = Date.now() - startTime;
+ return err && !result.overrideErr
+ ? resolve({ err, stdout, stderr })
+ : resolve(result);
+ })
+ );
+}
+
+// Makes a function that matches a RegExp to the process output, and then kills it
+// Optionally matches `matchCount` times, and edits `editFile` by appending `editString`
+export function makeMatchFn(
+ logger: Logger,
+ match: string,
+ matchCount: number,
+ editFile: string,
+ editString: string
+) {
+ return function (data: string, result: any, childProcess: NodeJS.Process) {
+ let localMatch = data.match(new RegExp(match));
+
+ logger.debug(`Match output: ${localMatch}`);
+
+ const killProcess = (childProcess: NodeJS.Process) => {
+ result.overrideErr = true;
+ treeKill(childProcess.pid);
+ };
+ if (localMatch) {
+ let firstMatch = localMatch[1];
+ if (!matchCount) {
+ result.match = firstMatch;
+ killProcess(childProcess);
+ } else {
+ result.counter = result.counter ? result.counter + 1 : 1;
+ if (result.counter < matchCount) {
+ if (editFile) {
+ fs.appendFileSync(editFile, editString);
+ }
+ } else {
+ result.match = firstMatch;
+ killProcess(childProcess);
+ }
+ }
+ }
+ };
+}
diff --git a/tools/bench/tsconfig.json b/tools/bench/tsconfig.json
new file mode 100644
index 000000000000..695ad82e479c
--- /dev/null
+++ b/tools/bench/tsconfig.json
@@ -0,0 +1,35 @@
+{
+ "compilerOptions": {
+ "declaration": true,
+ "experimentalDecorators": true,
+ "mapRoot": "",
+ "module": "commonjs",
+ "moduleResolution": "node",
+ "noEmitOnError": true,
+ "noImplicitAny": true,
+ "outDir": "../../dist/tools/bench",
+ "rootDir": ".",
+ "lib": [
+ "es2015",
+ "es6",
+ "dom"
+ ],
+ "skipLibCheck": true,
+ "target": "es5",
+ "sourceMap": true,
+ "sourceRoot": "/",
+ "baseUrl": ".",
+ "typeRoots": [
+ "../../node_modules/@types"
+ ],
+ "paths": {
+ "@ngtools/logger": [
+ "../../dist/@ngtools/logger/src"
+ ]
+ },
+ "types": [
+ "node",
+ "yargs"
+ ]
+ }
+}