diff --git a/.travis.yml b/.travis.yml index d9813c298474..73e3c4d84d12 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,7 @@ matrix: allow_failures: - env: NODE_SCRIPT="tests/run_e2e.js --nightly" - env: NODE_SCRIPT="tests/run_e2e.js --ng2" + - env: NODE_SCRIPT="tests/run_e2e.js --glob=benchmarks/**" - node_js: "7" include: - node_js: "6" @@ -40,6 +41,9 @@ matrix: - node_js: "6" os: linux env: NODE_SCRIPT="tests/run_e2e.js --nightly" + - node_js: "6" + os: linux + env: NODE_SCRIPT="tests/run_e2e.js --glob=benchmarks/**" - node_js: "7" os: linux env: NODE_SCRIPT=tests/run_e2e.js diff --git a/bin/ngbench b/bin/ngbench new file mode 100644 index 000000000000..dace66b43c76 --- /dev/null +++ b/bin/ngbench @@ -0,0 +1,8 @@ +#!/usr/bin/env node +'use strict'; + +// Provide a title to the process in `ps` +process.title = 'bench'; + +require('../lib/bootstrap-local'); +require('../tools/bench/bin/ngbench'); diff --git a/package.json b/package.json index 2fe90c297ba1..b9ab0ba39727 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,8 @@ "main": "packages/@angular/cli/lib/cli/index.js", "trackingCode": "UA-8594346-19", "bin": { - "ng": "./bin/ng" + "ng": "./bin/ng", + "ngbench": "./bin/ngbench" }, "keywords": [], "scripts": { @@ -126,6 +127,7 @@ "@types/semver": "^5.3.30", "@types/source-map": "^0.5.0", "@types/webpack": "^2.2.15", + "@types/yargs": "^6.5.0", "chai": "^3.5.0", "conventional-changelog": "^1.1.0", "dtsgenerator": "^0.9.1", @@ -145,10 +147,12 @@ "rewire": "^2.5.1", "sinon": "^1.17.3", "spdx-satisfies": "^0.1.3", + "strip-ansi": "^3.0.1", "through": "^2.3.6", - "tree-kill": "^1.0.0", + "tree-kill": "^1.1.0", "ts-node": "^2.0.0", - "tslint": "^5.1.0" + "tslint": "^5.1.0", + "yargs": "^6.6.0" }, "optionalDependencies": { "node-sass": "^4.3.0" diff --git a/tests/e2e/benchmarks/build.ts b/tests/e2e/benchmarks/build.ts new file mode 100644 index 000000000000..3398498408d1 --- /dev/null +++ b/tests/e2e/benchmarks/build.ts @@ -0,0 +1,6 @@ +import { ngbench } from '../utils/process'; + + +export default function () { + return ngbench('--command', 'ng build'); +} diff --git a/tests/e2e/benchmarks/serve/css.ts b/tests/e2e/benchmarks/serve/css.ts new file mode 100644 index 000000000000..eb57a66efb40 --- /dev/null +++ b/tests/e2e/benchmarks/serve/css.ts @@ -0,0 +1,42 @@ +import { ngbench } from '../../utils/process'; +import { replaceInFile, moveFile } from '../../utils/fs'; +import { updateJsonFile } from '../../utils/project'; + + +export default function () { + const extensions = ['css', 'scss', 'less', 'styl']; + let promise = Promise.resolve(); + + extensions.forEach(ext => { + promise = promise.then(() => { + // change files to use preprocessor + return updateJsonFile('.angular-cli.json', configJson => { + const app = configJson['apps'][0]; + app['styles'] = [`styles.${ext}`]; + }) + .then(() => replaceInFile('src/app/app.component.ts', + './app.component.css', `./app.component.${ext}`)) + .then(() => moveFile('src/styles.css', `src/styles.${ext}`)) + .then(() => moveFile('src/app/app.component.css', `src/app/app.component.${ext}`)) + // run benchmarks + .then(() => ngbench( + '--comment', 'component styles', + '--match-edit-file', `src/app/app.component.${ext}`, + '--match-edit-string', 'h1{color:blue}' + )) + .then(() => ngbench( + '--comment', `.${ext} CSS extension`, + '--match-edit-file', `src/styles.${ext}`, + '--match-edit-string', 'h1{color:blue}' + )) + // change files back + .then(() => replaceInFile('src/app/app.component.ts', + `./app.component.${ext}`, './app.component.css')) + .then(() => moveFile(`src/styles.${ext}`, 'src/styles.css')) + .then(() => moveFile(`src/app/app.component.${ext}`, 'src/app/app.component.css')); + + }); + }); + + return promise; +} diff --git a/tests/e2e/benchmarks/serve/html.ts b/tests/e2e/benchmarks/serve/html.ts new file mode 100644 index 000000000000..352e40b4fa2c --- /dev/null +++ b/tests/e2e/benchmarks/serve/html.ts @@ -0,0 +1,9 @@ +import { ngbench } from '../../utils/process'; + + +export default function () { + return ngbench( + '--match-edit-file', 'src/app/app.component.html', + '--match-edit-string', '
' + ); +} diff --git a/tests/e2e/benchmarks/serve/scripts.ts b/tests/e2e/benchmarks/serve/scripts.ts new file mode 100644 index 000000000000..11aee8b6f5e6 --- /dev/null +++ b/tests/e2e/benchmarks/serve/scripts.ts @@ -0,0 +1,16 @@ +import { ngbench } from '../../utils/process'; +import { writeFile } from '../../utils/fs'; +import { updateJsonFile } from '../../utils/project'; + + +export default function () { + return updateJsonFile('.angular-cli.json', configJson => { + const app = configJson['apps'][0]; + app['scripts'] = ['scripts.js']; + }) + .then(() => writeFile('src/scripts.js', '')) + .then(() => ngbench( + '--match-edit-file', 'src/scripts.js', + '--match-edit-string', 'console.log(1);' + )); +} diff --git a/tests/e2e/benchmarks/serve/serve.ts b/tests/e2e/benchmarks/serve/serve.ts new file mode 100644 index 000000000000..cfc92583f61d --- /dev/null +++ b/tests/e2e/benchmarks/serve/serve.ts @@ -0,0 +1,6 @@ +import { ngbench } from '../../utils/process'; + + +export default function () { + return ngbench('--extra-args=--aot'); +} diff --git a/tests/e2e/benchmarks/version.ts b/tests/e2e/benchmarks/version.ts new file mode 100644 index 000000000000..e429291699c9 --- /dev/null +++ b/tests/e2e/benchmarks/version.ts @@ -0,0 +1,6 @@ +import { ngbench } from '../utils/process'; + + +export default function () { + return ngbench('--command', 'ng version'); +} diff --git a/tests/e2e/utils/process.ts b/tests/e2e/utils/process.ts index 32ff695efaac..5ca0d0f65782 100644 --- a/tests/e2e/utils/process.ts +++ b/tests/e2e/utils/process.ts @@ -1,5 +1,6 @@ import * as child_process from 'child_process'; import {blue, yellow} from 'chalk'; +import {join} from 'path'; import {getGlobalVariable} from './env'; import {rimraf, writeFile} from './fs'; const treeKill = require('tree-kill'); @@ -32,7 +33,7 @@ function _exec(options: ExecOptions, cmd: string, args: string[]): Promise !!x) // Remove false and undefined. + .filter(x => !!x) // Remove false and undefined. .join(', ') .replace(/^(.+)$/, ' [$1]'); // Proper formatting. @@ -185,3 +186,8 @@ export function git(...args: string[]) { export function silentGit(...args: string[]) { return _exec({silent: true}, 'git', args); } + +export function ngbench(...args: string[]) { + const ngbenchJs = join(__dirname, '../../../bin/ngbench'); + return _exec({}, 'node', [ngbenchJs, ...args]); +} diff --git a/tools/bench/bin/ngbench b/tools/bench/bin/ngbench new file mode 100644 index 000000000000..3a54e2fe01bd --- /dev/null +++ b/tools/bench/bin/ngbench @@ -0,0 +1,2 @@ +#!/usr/bin/env node +require('../src/index'); diff --git a/tools/bench/package.json b/tools/bench/package.json new file mode 100644 index 000000000000..d1d2ebecdfab --- /dev/null +++ b/tools/bench/package.json @@ -0,0 +1,20 @@ +{ + "name": "bench", + "version": "0.0.0", + "private": false, + "description": "", + "main": "./src/index.js", + "bin": { + "ngbench": "./bin/ngbench" + }, + "license": "MIT", + "dependencies": { + "@ngtools/logger": "^0.1.4", + "chalk": "^1.1.3", + "lodash": "^4.11.1", + "rxjs": "^5.0.1", + "strip-ansi": "^3.0.1", + "tree-kill": "^1.1.0", + "yargs": "^6.6.0" + } +} diff --git a/tools/bench/src/benchmark-options.ts b/tools/bench/src/benchmark-options.ts new file mode 100644 index 000000000000..7023f19f310f --- /dev/null +++ b/tools/bench/src/benchmark-options.ts @@ -0,0 +1,12 @@ +export interface BenchmarkOptions { + command: string; + iterations: number; + extraArgs: string[]; + match: string; + matchCount: number; + matchEditFile: string; + matchEditString: string; + comment: string; + logFile: string; + debug: boolean; +} diff --git a/tools/bench/src/benchmark.ts b/tools/bench/src/benchmark.ts new file mode 100644 index 000000000000..3fed13263e47 --- /dev/null +++ b/tools/bench/src/benchmark.ts @@ -0,0 +1,114 @@ +import { Logger } from '@ngtools/logger'; +import * as fs from 'fs'; +const mean = require('lodash/mean'); +const toNumber = require('lodash/toNumber'); + + +import { BenchmarkOptions } from './benchmark-options'; +import { + combine, + makeMatchFn, + matchSpawn, + serialMultiPromise +} from './utils'; + +export function benchmark(benchmarkOptions: BenchmarkOptions, logger: Logger) { + const logIfDefined = (str: string, prop: any) => { + if (Array.isArray(prop) && prop.length === 0) { + prop = null; + } + return prop ? logger.info(str + prop) : null; + }; + + const cwd = process.cwd(); + + // Build match function + let matchFn: any = null; + if (benchmarkOptions.match) { + matchFn = makeMatchFn( + logger, + benchmarkOptions.match, + benchmarkOptions.matchCount, + benchmarkOptions.matchEditFile, + benchmarkOptions.matchEditString, + ); + } + + let editedFileContents: string; + if (benchmarkOptions.matchEditFile) { + // backup contents of file that is being edited for rebuilds + editedFileContents = fs.readFileSync(benchmarkOptions.matchEditFile, 'utf8'); + } + // combine flags commands + let flagCombinations = combine(benchmarkOptions.extraArgs); + flagCombinations.unshift([]); + + const startTime = Date.now(); + logger.info(`Base command: ${benchmarkOptions.command}`); + logger.info(`Iterations: ${benchmarkOptions.iterations}`); + logIfDefined('Comment: ', benchmarkOptions.comment); + logIfDefined('Extra args: ', benchmarkOptions.extraArgs); + logIfDefined('Logging to: ', benchmarkOptions.logFile); + if (benchmarkOptions.match) { + logger.info(`Match output: ${benchmarkOptions.match}`); + logIfDefined('Match count: ', benchmarkOptions.matchCount); + logIfDefined('Match edit file: ', benchmarkOptions.matchEditFile); + logIfDefined('Match edit string: ', benchmarkOptions.matchEditString); + } + if (benchmarkOptions.debug) { + logger.debug('### Debug mode, all output is logged ###'); + } + logger.info(''); + + + let promise = Promise.resolve(); + let hasFailures = false; + + flagCombinations.forEach((flags) => + promise = promise + .then(() => serialMultiPromise( + benchmarkOptions.iterations, + matchSpawn, + logger, + cwd, + matchFn, + benchmarkOptions.command, + flags + ).then((results: any[]) => { + const failures = results.filter(result => result.err && result.err !== 0); + logger.info(`Full command: ${benchmarkOptions.command} ${flags.join(' ')}`); + + let times = results.filter(result => !result.err) + .map((result) => result.time); + logger.info(`Time average: ${mean(times)}`); + logger.info(`Times: ${times.join()}`); + + if (benchmarkOptions.match) { + let matches = results.filter(result => !result.err) + .map((result) => result.match); + + let matchesAsNumbers = matches.map(toNumber); + if (matches.every(match => match != NaN)) { + logger.info(`Match average: ${mean(matchesAsNumbers)}`); + } + logger.info(`Matches: ${matches.join()}`); + } + + if (failures.length > 0) { + hasFailures = true; + logger.info(`Failures: ${failures.length}`); + logger.info(JSON.stringify(failures)); + } + logger.info(''); + })) + ); + + return promise.then(() => { + logger.info(`Benchmark execution time: ${Date.now() - startTime}ms`); + // restore contents of file that was being edited for rebuilds + if (benchmarkOptions.matchEditFile) { + fs.writeFileSync(benchmarkOptions.matchEditFile, editedFileContents, 'utf8'); + } + return hasFailures ? Promise.reject(new Error('Some benchmarks failed')) : Promise.resolve(); + }); +} diff --git a/tools/bench/src/index.ts b/tools/bench/src/index.ts new file mode 100644 index 000000000000..ecb26fb787d9 --- /dev/null +++ b/tools/bench/src/index.ts @@ -0,0 +1,126 @@ +import { Logger, LogEntry } from '@ngtools/logger'; +import { bold, red, yellow, white } from 'chalk'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as yargs from 'yargs'; +import 'rxjs/add/operator/filter'; + +import { BenchmarkOptions } from './benchmark-options'; +import { benchmark } from './benchmark'; + + +// Set options via yargs +const benchmarkOptions: BenchmarkOptions = yargs + .usage('$0 [args]') + .options({ + 'command': { + description: 'Command to benchmark, defaults to benchmarking `ng serve` if not set', + type: 'string', + alias: 'c' + }, + 'iterations': { + description: 'Number of iterations to run benchmark', + type: 'number', + default: 5, + alias: 'i' + }, + 'extra-args': { + description: 'Extra arguments to combine and run', + type: 'array', + default: [], + alias: 'ea' + }, + 'match': { + description: 'Command output to match', + type: 'string', + alias: 'm' + }, + 'match-count': { + description: 'Times to match output', + type: 'number', + alias: 'mc' + }, + 'match-edit-file': { + description: 'File to edit after a output match', + type: 'string', + alias: 'mef' + }, + 'match-edit-string': { + description: 'String to use in --match-edit-file', + type: 'string', + alias: 'mes' + }, + 'comment': { + description: 'Comment to add to output', + type: 'string', + alias: 'cm' + }, + 'log-file': { + description: 'File to log output', + type: 'string', + alias: 'lf' + }, + 'debug': { + description: 'Show command output', + type: 'boolean', + default: false, + alias: 'd' + }, + }) + .help() + .argv; + +// Initialize logger +const logger = new Logger('ngbench'); + +logger + .filter((entry: LogEntry) => (entry.level != 'debug' || benchmarkOptions.debug)) + .subscribe((entry: LogEntry) => { + let color: (s: string) => string = white; + let output = process.stdout; + switch (entry.level) { + case 'info': color = white; break; + case 'warn': color = yellow; break; + case 'error': color = red; output = process.stderr; break; + case 'fatal': color = (x: string) => bold(red(x)); output = process.stderr; break; + } + + output.write(color(entry.message) + '\n'); + if (benchmarkOptions.logFile) { + fs.appendFileSync(benchmarkOptions.logFile, entry.message + '\n'); + } + }); + +logger + .filter((entry: LogEntry) => entry.level == 'fatal') + .subscribe(() => { + process.stderr.write('A fatal error happened. See details above.'); + process.exit(1); + }); + + +// Set compound defauls and resolve paths +if (!benchmarkOptions.command) { + benchmarkOptions.command = 'ng serve --no-progress'; + benchmarkOptions.match = benchmarkOptions.match || 'Time: (.*)ms'; + benchmarkOptions.matchCount = benchmarkOptions.matchCount || 4; + benchmarkOptions.matchEditFile = benchmarkOptions.matchEditFile || 'src/main.ts'; + benchmarkOptions.matchEditString = benchmarkOptions.matchEditString || 'console.log(1);'; +} + +if (benchmarkOptions.matchEditFile) { + benchmarkOptions.matchEditFile = path.resolve('./', benchmarkOptions.matchEditFile); +} + +if (benchmarkOptions.command.match(/^ng (build|serve)/) + && benchmarkOptions.command.match(/--no-progress/) === null +) { + logger.warn('Auto-added \'--no-progress\' to build/serve command.'); + benchmarkOptions.command = benchmarkOptions.command + ' --no-progress'; +} + +// Run benchmark +benchmark(benchmarkOptions, logger) + .catch((err) => { + logger.fatal(JSON.stringify(err)); + }); diff --git a/tools/bench/src/utils.ts b/tools/bench/src/utils.ts new file mode 100644 index 000000000000..a0ced018e812 --- /dev/null +++ b/tools/bench/src/utils.ts @@ -0,0 +1,138 @@ +import * as fs from 'fs'; +import * as child_process from 'child_process'; +import { Logger } from '@ngtools/logger'; +const stripAnsi = require('strip-ansi'); +const treeKill = require('tree-kill'); + +// Make all combinations of array elements +// http://stackoverflow.com/a/5752056/2116927 +export function combine(array: any[]) { + let fn = function (n: number, src: any[], got: any[], all: any[]) { + if (n == 0) { + if (got.length > 0) { + all[all.length] = got; + } + return; + } + for (let j = 0; j < src.length; j++) { + fn(n - 1, src.slice(j + 1), got.concat([src[j]]), all); + } + return; + }; + let all: any[] = []; + for (let i = 0; i < array.length; i++) { + fn(i, array, [], all); + } + if (array.length > 0) { all.push(array); } + return all; +} + +// Returns a promise with the result of calling `times` times the `fn` function with `args` +// `fn` will be called in serial, and is expected to return a promise +export function serialMultiPromise(times: number, fn: any, ...args: any[]) { + let results: Promise[] = []; + let promise = Promise.resolve(); + Array.from({ length: times }).forEach(() => promise = promise.then(() => + fn.apply(null, args).then((result: Promise) => results.push(result)) + )); + return promise.then(() => results); +} + +// Spawns `cmd` with `args` +// Calls `matchFn` with the `stdout` output, a `result` var and the process +// `dataFn` is expected to modify `result` +export function matchSpawn(logger: Logger, cwd: string, matchFn: any, + cmd: string, args: string[] = []) { + // dataFn will have access to result and use it to store results + let result = { + // overrideErr will signal that an error code on the exit event should be ignored + // This is useful on windows where killing a tree of processes always makes them + // exit with an error code + overrideErr: false, + time: 0 + }; + let stdout = ''; + let stderr = ''; + let spawnOptions: any = { + cwd: cwd, + env: process.env + }; + + // Split given `cmd` and pass on any args to `args` + const splitCmd = cmd.split(' '); + cmd = splitCmd.shift(); + args = splitCmd.concat(args); + + if (process.platform.startsWith('win')) { + args = ['/c', cmd].concat(args); + cmd = 'cmd.exe'; + spawnOptions['stdio'] = 'pipe'; + } + + logger.debug(`spawning cmd: ${cmd}, args: ${args}, cwd: ${cwd}`); + + const startTime = Date.now(); + const childProcess = child_process.spawn(cmd, args, spawnOptions); + + childProcess.stdout.on('data', (data: Buffer) => { + let strippedData = stripAnsi(data.toString('utf-8')); + logger.debug(strippedData); + stdout += strippedData; + if (matchFn) { + matchFn(strippedData, result, childProcess); + } + }); + + childProcess.stderr.on('data', (data: Buffer) => { + let strippedData = stripAnsi(data.toString('utf-8')); + logger.debug(strippedData); + stderr += strippedData; + }); + + return new Promise((resolve, _) => + childProcess.on('exit', (err: number) => { + result.time = Date.now() - startTime; + return err && !result.overrideErr + ? resolve({ err, stdout, stderr }) + : resolve(result); + }) + ); +} + +// Makes a function that matches a RegExp to the process output, and then kills it +// Optionally matches `matchCount` times, and edits `editFile` by appending `editString` +export function makeMatchFn( + logger: Logger, + match: string, + matchCount: number, + editFile: string, + editString: string +) { + return function (data: string, result: any, childProcess: NodeJS.Process) { + let localMatch = data.match(new RegExp(match)); + + logger.debug(`Match output: ${localMatch}`); + + const killProcess = (childProcess: NodeJS.Process) => { + result.overrideErr = true; + treeKill(childProcess.pid); + }; + if (localMatch) { + let firstMatch = localMatch[1]; + if (!matchCount) { + result.match = firstMatch; + killProcess(childProcess); + } else { + result.counter = result.counter ? result.counter + 1 : 1; + if (result.counter < matchCount) { + if (editFile) { + fs.appendFileSync(editFile, editString); + } + } else { + result.match = firstMatch; + killProcess(childProcess); + } + } + } + }; +} diff --git a/tools/bench/tsconfig.json b/tools/bench/tsconfig.json new file mode 100644 index 000000000000..695ad82e479c --- /dev/null +++ b/tools/bench/tsconfig.json @@ -0,0 +1,35 @@ +{ + "compilerOptions": { + "declaration": true, + "experimentalDecorators": true, + "mapRoot": "", + "module": "commonjs", + "moduleResolution": "node", + "noEmitOnError": true, + "noImplicitAny": true, + "outDir": "../../dist/tools/bench", + "rootDir": ".", + "lib": [ + "es2015", + "es6", + "dom" + ], + "skipLibCheck": true, + "target": "es5", + "sourceMap": true, + "sourceRoot": "/", + "baseUrl": ".", + "typeRoots": [ + "../../node_modules/@types" + ], + "paths": { + "@ngtools/logger": [ + "../../dist/@ngtools/logger/src" + ] + }, + "types": [ + "node", + "yargs" + ] + } +}