benchmark: collect memory usage per operation (#2088)

Author: IvanGoncharov

package.json

@@ -28,7 +28,7 @@
     "testonly": "mocha --full-trace src/**/__tests__/**/*-test.js",
     "testonly:cover": "nyc npm run testonly",
     "lint": "eslint --cache --report-unused-disable-directives src resources",
-    "benchmark": "node --predictable ./resources/benchmark.js",
+    "benchmark": "node --noconcurrent_sweeping --expose-gc --predictable ./resources/benchmark.js",
     "prettier": "prettier --ignore-path .gitignore --write --list-different \"**/*.{js,md,json,yml}\"",
     "prettier:check": "prettier --ignore-path .gitignore --check \"**/*.{js,md,json,yml}\"",
     "check": "flow check",

resources/benchmark-fork.js

@@ -11,7 +11,7 @@ function clock(count, fn) {
   for (let i = 0; i < count; ++i) {
     fn();
   }
-  return Number(process.hrtime.bigint() - start) / count;
+  return Number(process.hrtime.bigint() - start);
 }
 
 if (require.main === module) {
@@ -21,10 +21,14 @@ if (require.main === module) {
   const module = require(modulePath);
 
   clock(7, module.measure); // warm up
+  global.gc();
   process.nextTick(() => {
+    const memBaseline = process.memoryUsage().heapUsed;
+    const clocked = clock(module.count, module.measure);
     process.send({
       name: module.name,
-      clocked: clock(module.count, module.measure),
+      clocked: clocked / module.count,
+      memUsed: (process.memoryUsage().heapUsed - memBaseline) / module.count,
     });
   });
 }
@@ -44,6 +48,9 @@ function sampleModule(modulePath) {
       }
       reject(error || new Error('Forked process closed without error'));
     });
+  }).then(result => {
+    global.gc();
+    return result;
   });
 }
 

resources/benchmark.js

@@ -102,9 +102,10 @@ async function collectSamples(modulePath) {
   // If time permits, increase sample size to reduce the margin of error.
   const start = Date.now();
   while (samples.length < minSamples || (Date.now() - start) / 1e3 < maxTime) {
-    const { clocked } = await sampleModule(modulePath);
+    const { clocked, memUsed } = await sampleModule(modulePath);
     assert(clocked > 0);
-    samples.push(clocked);
+    assert(memUsed > 0);
+    samples.push({ clocked, memUsed });
   }
   return samples;
 }
@@ -126,15 +127,18 @@ function computeStats(samples) {
 
   // Compute the sample mean (estimate of the population mean).
   let mean = 0;
-  for (const x of samples) {
-    mean += x;
+  let meanMemUsed = 0;
+  for (const { clocked, memUsed } of samples) {
+    mean += clocked;
+    meanMemUsed += memUsed;
   }
   mean /= samples.length;
+  meanMemUsed /= samples.length;
 
   // Compute the sample variance (estimate of the population variance).
   let variance = 0;
-  for (const x of samples) {
-    variance += Math.pow(x - mean, 2);
+  for (const { clocked } of samples) {
+    variance += Math.pow(clocked - mean, 2);
   }
   variance /= samples.length - 1;
 
@@ -157,22 +161,28 @@ function computeStats(samples) {
   const rme = (moe / mean) * 100 || 0;
 
   return {
+    memPerOp: Math.floor(meanMemUsed),
     ops: NS_PER_SEC / mean,
     deviation: rme,
+    numSamples: samples.length,
   };
 }
 
 function beautifyBenchmark(results) {
   const nameMaxLen = maxBy(results, ({ name }) => name.length);
   const opsTop = maxBy(results, ({ ops }) => ops);
   const opsMaxLen = maxBy(results, ({ ops }) => beautifyNumber(ops).length);
+  const memPerOpMaxLen = maxBy(
+    results,
+    ({ memPerOp }) => beautifyBytes(memPerOp).length,
+  );
 
   for (const result of results) {
     printBench(result);
   }
 
   function printBench(bench) {
-    const { name, ops, deviation, samples } = bench;
+    const { name, memPerOp, ops, deviation, numSamples } = bench;
     console.log(
       '  ' +
         nameStr() +
@@ -182,7 +192,10 @@ function beautifyBenchmark(results) {
         grey('\xb1') +
         deviationStr() +
         cyan('%') +
-        grey(' (' + samples.length + ' runs sampled)'),
+        grey(' x ') +
+        memPerOpStr() +
+        '/op' +
+        grey(' (' + numSamples + ' runs sampled)'),
     );
 
     function nameStr() {
@@ -200,9 +213,19 @@ function beautifyBenchmark(results) {
       const colorFn = deviation > 5 ? red : deviation > 2 ? yellow : green;
       return colorFn(deviation.toFixed(2));
     }
+
+    function memPerOpStr() {
+      return beautifyBytes(memPerOp).padStart(memPerOpMaxLen);
+    }
   }
 }
 
+function beautifyBytes(bytes) {
+  const sizes = ['Bytes', 'KB', 'MB', 'GB'];
+  const i = Math.floor(Math.log2(bytes) / 10);
+  return beautifyNumber(bytes / Math.pow(2, i * 10)) + ' ' + sizes[i];
+}
+
 function beautifyNumber(num) {
   return Number(num.toFixed(num > 100 ? 0 : 2)).toLocaleString();
 }