test: add benchmark tests

filipesilva · filipesilva · commit 9ce843fd09fb · 2017-05-23T11:58:13.000+01:00
Adds `tools/bench` allowing the benchmark of commands via the `ngbench` binary in development environments.
diff --git a/.travis.yml b/.travis.yml
@@ -12,6 +12,7 @@ matrix:
   allow_failures:
     - env: NODE_SCRIPT="tests/run_e2e.js --nightly"
     - env: NODE_SCRIPT="tests/run_e2e.js --ng2"
+    - env: NODE_SCRIPT="tests/run_e2e.js --glob=benchmarks/**"
     - node_js: "7"
   include:
     - node_js: "6"
@@ -40,6 +41,9 @@ matrix:
     - node_js: "6"
       os: linux
       env: NODE_SCRIPT="tests/run_e2e.js --nightly"
+    - node_js: "6"
+      os: linux
+      env: NODE_SCRIPT="tests/run_e2e.js --glob=benchmarks/**"
     - node_js: "7"
       os: linux
       env: NODE_SCRIPT=tests/run_e2e.js
diff --git a/bin/ngbench b/bin/ngbench
@@ -0,0 +1,8 @@
+#!/usr/bin/env node
+'use strict';
+
+// Provide a title to the process in `ps`
+process.title = 'bench';
+
+require('../lib/bootstrap-local');
+require('../tools/bench/bin/ngbench');
diff --git a/package.json b/package.json
@@ -5,7 +5,8 @@
   "main": "packages/@angular/cli/lib/cli/index.js",
   "trackingCode": "UA-8594346-19",
   "bin": {
-    "ng": "./bin/ng"
+    "ng": "./bin/ng",
+    "ngbench": "./bin/ngbench"
   },
   "keywords": [],
   "scripts": {
@@ -87,18 +88,21 @@
     "silent-error": "^1.0.0",
     "source-map": "^0.5.6",
     "source-map-loader": "^0.2.0",
+    "strip-ansi": "^3.0.1",
     "style-loader": "^0.13.1",
     "stylus": "^0.54.5",
     "stylus-loader": "^3.0.1",
     "temp": "0.8.3",
     "typescript": "~2.3.1",
+    "tree-kill": "^1.1.0",
     "url-loader": "^0.5.7",
     "walk-sync": "^0.3.1",
     "webpack": "~2.4.0",
     "webpack-dev-middleware": "^1.10.2",
     "webpack-dev-server": "~2.4.5",
     "webpack-merge": "^2.4.0",
-    "zone.js": "^0.8.4"
+    "zone.js": "^0.8.4",
+    "yargs": "^6.6.0"
   },
   "ember-addon": {
     "paths": [
@@ -126,6 +130,7 @@
     "@types/semver": "^5.3.30",
     "@types/source-map": "^0.5.0",
     "@types/webpack": "^2.2.15",
+    "@types/yargs": "^6.5.0",
     "chai": "^3.5.0",
     "conventional-changelog": "^1.1.0",
     "dtsgenerator": "^0.9.1",
@@ -146,7 +151,7 @@
     "sinon": "^1.17.3",
     "spdx-satisfies": "^0.1.3",
     "through": "^2.3.6",
-    "tree-kill": "^1.0.0",
+    "tree-kill": "^1.1.0",
     "ts-node": "^2.0.0",
     "tslint": "^5.1.0"
   },
diff --git a/tests/e2e/benchmarks/build.ts b/tests/e2e/benchmarks/build.ts
@@ -0,0 +1,6 @@
+import { ngbench } from '../utils/process';
+
+
+export default function () {
+  return ngbench('--command', 'ng build');
+}
diff --git a/tests/e2e/benchmarks/serve/css.ts b/tests/e2e/benchmarks/serve/css.ts
@@ -0,0 +1,42 @@
+import { ngbench } from '../../utils/process';
+import { replaceInFile, moveFile } from '../../utils/fs';
+import { updateJsonFile } from '../../utils/project';
+
+
+export default function () {
+  const extensions = ['css', 'scss', 'less', 'styl'];
+  let promise = Promise.resolve();
+
+  extensions.forEach(ext => {
+    promise = promise.then(() => {
+      // change files to use preprocessor
+      return updateJsonFile('.angular-cli.json', configJson => {
+        const app = configJson['apps'][0];
+        app['styles'] = [`styles.${ext}`];
+      })
+        .then(() => replaceInFile('src/app/app.component.ts',
+          './app.component.css', `./app.component.${ext}`))
+        .then(() => moveFile('src/styles.css', `src/styles.${ext}`))
+        .then(() => moveFile('src/app/app.component.css', `src/app/app.component.${ext}`))
+        // run benchmarks
+        .then(() => ngbench(
+          '--comment', 'component styles',
+          '--match-edit-file', `src/app/app.component.${ext}`,
+          '--match-edit-string', 'h1{color:blue}'
+        ))
+        .then(() => ngbench(
+          '--comment', `.${ext} CSS extension`,
+          '--match-edit-file', `src/styles.${ext}`,
+          '--match-edit-string', 'h1{color:blue}'
+        ))
+        // change files back
+        .then(() => replaceInFile('src/app/app.component.ts',
+          `./app.component.${ext}`, './app.component.css'))
+        .then(() => moveFile(`src/styles.${ext}`, 'src/styles.css'))
+        .then(() => moveFile(`src/app/app.component.${ext}`, 'src/app/app.component.css'));
+
+    });
+  });
+
+  return promise;
+}
diff --git a/tests/e2e/benchmarks/serve/html.ts b/tests/e2e/benchmarks/serve/html.ts
@@ -0,0 +1,9 @@
+import { ngbench } from '../../utils/process';
+
+
+export default function () {
+  return ngbench(
+    '--match-edit-file', 'src/app/app.component.html',
+    '--match-edit-string', '<div></div>'
+  );
+}
diff --git a/tests/e2e/benchmarks/serve/scripts.ts b/tests/e2e/benchmarks/serve/scripts.ts
@@ -0,0 +1,16 @@
+import { ngbench } from '../../utils/process';
+import { writeFile } from '../../utils/fs';
+import { updateJsonFile } from '../../utils/project';
+
+
+export default function () {
+  return updateJsonFile('.angular-cli.json', configJson => {
+    const app = configJson['apps'][0];
+    app['scripts'] = ['scripts.js'];
+  })
+    .then(() => writeFile('src/scripts.js', ''))
+    .then(() => ngbench(
+      '--match-edit-file', 'src/scripts.js',
+      '--match-edit-string', 'console.log(1);'
+    ));
+}
diff --git a/tests/e2e/benchmarks/serve/serve.ts b/tests/e2e/benchmarks/serve/serve.ts
@@ -0,0 +1,6 @@
+import { ngbench } from '../../utils/process';
+
+
+export default function () {
+  return ngbench('--extra-args=--aot');
+}
diff --git a/tests/e2e/benchmarks/version.ts b/tests/e2e/benchmarks/version.ts
@@ -0,0 +1,6 @@
+import { ngbench } from '../utils/process';
+
+
+export default function () {
+  return ngbench('--command', 'ng version');
+}
diff --git a/tests/e2e/utils/process.ts b/tests/e2e/utils/process.ts
@@ -1,5 +1,6 @@
 import * as child_process from 'child_process';
 import {blue, yellow} from 'chalk';
+import {join} from 'path';
 import {getGlobalVariable} from './env';
 import {rimraf, writeFile} from './fs';
 const treeKill = require('tree-kill');
@@ -185,3 +186,8 @@ export function git(...args: string[]) {
 export function silentGit(...args: string[]) {
   return _exec({silent: true}, 'git', args);
 }
+
+export function ngbench(...args: string[]) {
+  const ngbenchBin = join(__dirname, '../../../bin/ngbench');
+  return _exec({}, ngbenchBin, args);
+}
diff --git a/tools/bench/bin/ngbench b/tools/bench/bin/ngbench
@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+require('../src/index');
diff --git a/tools/bench/package.json b/tools/bench/package.json
@@ -0,0 +1,20 @@
+{
+  "name": "bench",
+  "version": "0.0.0",
+  "private": false,
+  "description": "",
+  "main": "./src/index.js",
+  "bin": {
+    "ngbench": "./bin/ngbench"
+  },
+  "license": "MIT",
+  "dependencies": {
+    "@ngtools/logger": "^0.1.4",
+    "chalk": "^1.1.3",
+    "lodash": "^4.11.1",
+    "rxjs": "^5.0.1",
+    "strip-ansi": "^3.0.1",
+    "tree-kill": "^1.1.0",
+    "yargs": "^6.6.0"
+  }
+}
diff --git a/tools/bench/src/benchmark-options.ts b/tools/bench/src/benchmark-options.ts
@@ -0,0 +1,12 @@
+export interface BenchmarkOptions {
+  command: string;
+  iterations: number;
+  extraArgs: string[];
+  match: string;
+  matchCount: number;
+  matchEditFile: string;
+  matchEditString: string;
+  comment: string;
+  logFile: string;
+  debug: boolean;
+}
diff --git a/tools/bench/src/benchmark.ts b/tools/bench/src/benchmark.ts
@@ -0,0 +1,114 @@
+import { Logger } from '@ngtools/logger';
+import * as fs from 'fs';
+const mean = require('lodash/mean');
+const toNumber = require('lodash/toNumber');
+
+
+import { BenchmarkOptions } from './benchmark-options';
+import {
+  combine,
+  makeMatchFn,
+  matchSpawn,
+  serialMultiPromise
+} from './utils';
+
+export function benchmark(benchmarkOptions: BenchmarkOptions, logger: Logger) {
+  const logIfDefined = (str: string, prop: any) => {
+    if (Array.isArray(prop) && prop.length === 0) {
+      prop = null;
+    }
+    return prop ? logger.info(str + prop) : null;
+  };
+
+  const cwd = process.cwd();
+
+  // Build match function
+  let matchFn: any = null;
+  if (benchmarkOptions.match) {
+    matchFn = makeMatchFn(
+      logger,
+      benchmarkOptions.match,
+      benchmarkOptions.matchCount,
+      benchmarkOptions.matchEditFile,
+      benchmarkOptions.matchEditString,
+    );
+  }
+
+  let editedFileContents: string;
+  if (benchmarkOptions.matchEditFile) {
+    // backup contents of file that is being edited for rebuilds
+    editedFileContents = fs.readFileSync(benchmarkOptions.matchEditFile, 'utf8');
+  }
+  // combine flags commands
+  let flagCombinations = combine(benchmarkOptions.extraArgs);
+  flagCombinations.unshift([]);
+
+  const startTime = Date.now();
+  logger.info(`Base command: ${benchmarkOptions.command}`);
+  logger.info(`Iterations: ${benchmarkOptions.iterations}`);
+  logIfDefined('Comment: ', benchmarkOptions.comment);
+  logIfDefined('Extra args: ', benchmarkOptions.extraArgs);
+  logIfDefined('Logging to: ', benchmarkOptions.logFile);
+  if (benchmarkOptions.match) {
+    logger.info(`Match output: ${benchmarkOptions.match}`);
+    logIfDefined('Match count: ', benchmarkOptions.matchCount);
+    logIfDefined('Match edit file: ', benchmarkOptions.matchEditFile);
+    logIfDefined('Match edit string: ', benchmarkOptions.matchEditString);
+  }
+  if (benchmarkOptions.debug) {
+    logger.debug('### Debug mode, all output is logged ###');
+  }
+  logger.info('');
+
+
+  let promise = Promise.resolve();
+  let hasFailures = false;
+
+  flagCombinations.forEach((flags) =>
+    promise = promise
+      .then(() => serialMultiPromise(
+        benchmarkOptions.iterations,
+        matchSpawn,
+        logger,
+        cwd,
+        matchFn,
+        benchmarkOptions.command,
+        flags
+      ).then((results: any[]) => {
+        const failures = results.filter(result => result.err && result.err !== 0);
+        logger.info(`Full command: ${benchmarkOptions.command} ${flags.join(' ')}`);
+
+        let times = results.filter(result => !result.err)
+          .map((result) => result.time);
+        logger.info(`Time average: ${mean(times)}`);
+        logger.info(`Times: ${times.join()}`);
+
+        if (benchmarkOptions.match) {
+          let matches = results.filter(result => !result.err)
+            .map((result) => result.match);
+
+          let matchesAsNumbers = matches.map(toNumber);
+          if (matches.every(match => match != NaN)) {
+            logger.info(`Match average: ${mean(matchesAsNumbers)}`);
+          }
+          logger.info(`Matches: ${matches.join()}`);
+        }
+
+        if (failures.length > 0) {
+          hasFailures = true;
+          logger.info(`Failures: ${failures.length}`);
+          logger.info(JSON.stringify(failures));
+        }
+        logger.info('');
+      }))
+  );
+
+  return promise.then(() => {
+    logger.info(`Benchmark execution time: ${Date.now() - startTime}ms`);
+    // restore contents of file that was being edited for rebuilds
+    if (benchmarkOptions.matchEditFile) {
+      fs.writeFileSync(benchmarkOptions.matchEditFile, editedFileContents, 'utf8');
+    }
+    return hasFailures ? Promise.reject(new Error('Some benchmarks failed')) : Promise.resolve();
+  });
+}
diff --git a/tools/bench/src/index.ts b/tools/bench/src/index.ts
diff --git a/tools/bench/src/utils.ts b/tools/bench/src/utils.ts
diff --git a/tools/bench/tsconfig.json b/tools/bench/tsconfig.json
