Skip to content

Commit bc95ab4

Browse files
W-A-Jamesdurran
W-A-James
and
durran
authored
chore(NODE-5362): update benchmark tooling (#584)
Co-authored-by: Durran Jordan <[email protected]>
1 parent b0898fb commit bc95ab4

File tree

4 files changed

+119
-52
lines changed

4 files changed

+119
-52
lines changed

‎etc/benchmarks/bson_versions.json‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
{
2+
"versions": [
3+
"1.1.6",
4+
"4.6",
5+
"5.0",
6+
"5.1",
7+
"5.2",
8+
"5.3"
9+
]
10+
}

‎etc/benchmarks/convert_to_csv.sh‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
#!/bin/bash
2+
# This script is meant to be used on the output of benchmark runs to generate a csv file that can be
3+
# more easily ingested in google sheets or your spreadsheet/data anaylsis tool of choice
4+
# note that you will also see the output of the csv file printed in the terminal
5+
USAGE=$(/bin/cat <<EOM
6+
Usage:
7+
./convert_to_csv.sh <input> [<output>]
8+
9+
Arguments:
10+
input - file to read from
11+
output - file to output csv (if not provided defaults to 'results.csv')
12+
EOM
13+
)
14+
INPUT=$1
15+
OUTPUT=${2:-results.csv}
16+
SED_SCRIPT=$(cat <<EOM
17+
# filter for lines that contain the max field
18+
/^.*max.*\$/!d
19+
20+
# remove spaces
21+
s/ //g
22+
23+
# replace pipes with commas
24+
y/|/,/
25+
26+
# remove trailing and leading comma
27+
s/^,(.*),\$/\1/
28+
29+
# remove field names
30+
s/([a-zA-Z0-9]+:)//g
31+
32+
# remove units
33+
s/([0-9]+)ms/\1/g
34+
35+
# split version and test
36+
s/---/,/
37+
P
38+
EOM
39+
)
40+
41+
if [ -z $INPUT ]; then
42+
echo "$USAGE"
43+
exit 1
44+
fi
45+
46+
47+
lines=$(sed --quiet --regexp-extended -e "$SED_SCRIPT" < $INPUT)
48+
49+
echo 'version,test,max,min,mean,stddev,p90,p95,p99' | tee $OUTPUT
50+
for line in $lines; do
51+
echo $line | tee -a $OUTPUT
52+
done

‎etc/benchmarks/install_bson_versions.sh‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
#!/bin/bash
2+
versions=$(jq '.versions' < bson_versions.json | sed -E 's/(\[|\]|,|")//g')
3+
installVersions=''
4+
for bson in $versions; do
5+
versionNoDot=$(echo $bson | tr -d '.')
6+
installVersions+=" bson${versionNoDot}@npm:bson@${bson}"
7+
done
8+
9+
set -o xtrace
10+
npm install --no-save ${installVersions}

‎etc/benchmarks/lib_runner.mjs‎

Lines changed: 47 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import { cpus, totalmem } from 'os';
66
import { exec as execCb } from 'child_process';
77
import { promisify, types } from 'util';
88
import { writeFile } from 'fs/promises';
9+
import * as semver from 'semver';
910
import v8Profiler from 'v8-profiler-next';
1011
import chalk from 'chalk';
1112
const exec = promisify(execCb);
@@ -53,67 +54,60 @@ export function getCurrentLocalBSON(libs) {
5354
}
5455

5556
export async function getLibs() {
56-
return await Promise.all([
57-
(async () => {
58-
const { stdout } = await exec('git rev-parse --short HEAD');
59-
const hash = stdout.trim();
57+
const bsonVersions = await readJSONFile('./bson_versions.json');
58+
const entries = bsonVersions.versions.map(async (version) => {
59+
const bsonPath = `../../node_modules/bson${version.replaceAll('.', '')}`;
60+
const packageVersion = (await readJSONFile(`${bsonPath}/package.json`)).version;
61+
if (semver.lte(semver.coerce(version), '3.0.0')) {
62+
const legacy = (await import(`${bsonPath}/index.js`)).default;
6063
return {
61-
name: 'local',
62-
lib: await import('../../lib/index.js'),
63-
version: hash
64+
name: version,
65+
lib: { ...legacy, ...legacy.prototype },
66+
version: packageVersion
6467
};
65-
})(),
66-
(async () => ({
67-
name: 'released',
68-
lib: await import('../../node_modules/bson_latest/lib/bson.js'),
69-
version: (await readJSONFile('../../node_modules/bson_latest/package.json')).version
70-
}))(),
71-
(async () => {
72-
const legacyBSON = (await import('../../node_modules/bson_legacy/index.js')).default;
68+
} else if (semver.gte(semver.coerce(version), '5.0.0')) {
7369
return {
74-
name: 'previous major',
75-
lib: { ...legacyBSON, ...legacyBSON.prototype },
76-
version: (await readJSONFile('../../node_modules/bson_legacy/package.json')).version
70+
name: version,
71+
lib: await import(`${bsonPath}/lib/bson.cjs`),
72+
version: packageVersion
7773
};
78-
})()
79-
// BSON-EXT is EOL so we do not need to keep testing it, and it has issues installing it
80-
// in this no-save way on M1 currently that are not worth fixing.
81-
// (async () => ({
82-
// name: 'bson-ext',
83-
// lib: await import('../../node_modules/bson_ext/lib/index.js'),
84-
// version: (await readJSONFile('../../node_modules/bson_ext/package.json')).version
85-
// }))()
86-
]).catch(error => {
87-
console.error(error);
88-
console.error(
89-
`Please run:\n${[
90-
'npm run build',
91-
'npm install --no-save bson_ext@npm:bson-ext@4 bson_legacy@npm:bson@1 bson_latest@npm:bson@latest'
92-
].join('\n')}`
93-
);
74+
} else {
75+
return {
76+
name: version,
77+
lib: await import(`${bsonPath}/lib/bson.js`),
78+
version: packageVersion
79+
};
80+
}
81+
});
82+
83+
entries.unshift({
84+
name: 'local',
85+
lib: await import('../../lib/bson.cjs'),
86+
version: (await readJSONFile('../../package.json')).version
87+
});
88+
89+
return await Promise.all(entries).catch(e => {
90+
console.error(e);
91+
console.error('Run\n\tnpm run build\n\t,./etc/benchmarks/install_bson_versions.sh');
9492
process.exit(1);
9593
});
9694
}
9795

9896
const printHistogram = (name, h) => {
9997
const makeReadableTime = nanoseconds => (nanoseconds / 1e6).toFixed(3).padStart(7, ' ');
100-
console.log();
101-
console.log(chalk.green(name));
102-
console.log('-'.repeat(155));
103-
process.stdout.write(`| ${chalk.cyan('max')}: ${chalk.red(makeReadableTime(h.max))} ms |`);
104-
process.stdout.write(` ${chalk.cyan('min')}: ${chalk.red(makeReadableTime(h.min))} ms |`);
105-
process.stdout.write(` ${chalk.cyan('mean')}: ${chalk.red(makeReadableTime(h.mean))} ms |`);
106-
process.stdout.write(` ${chalk.cyan('stddev')}: ${chalk.red(makeReadableTime(h.stddev))} ms |`);
107-
process.stdout.write(
108-
` ${chalk.cyan('p90th')}: ${chalk.red(makeReadableTime(h.percentile(90)))} ms |`
109-
);
110-
process.stdout.write(
111-
` ${chalk.cyan('p95th')}: ${chalk.red(makeReadableTime(h.percentile(95)))} ms |`
112-
);
113-
process.stdout.write(
98+
const line = [
99+
`| ${chalk.green(name.replaceAll(' ', '-'))} | ${chalk.cyan('max')}: ${chalk.red(makeReadableTime(h.max))} ms |`,
100+
` ${chalk.cyan('min')}: ${chalk.red(makeReadableTime(h.min))} ms |`,
101+
` ${chalk.cyan('mean')}: ${chalk.red(makeReadableTime(h.mean))} ms |`,
102+
` ${chalk.cyan('stddev')}: ${chalk.red(makeReadableTime(h.stddev))} ms |`,
103+
` ${chalk.cyan('p90th')}: ${chalk.red(makeReadableTime(h.percentile(90)))} ms |`,
104+
` ${chalk.cyan('p95th')}: ${chalk.red(makeReadableTime(h.percentile(95)))} ms |`,
114105
` ${chalk.cyan('p99th')}: ${chalk.red(makeReadableTime(h.percentile(99)))} ms |`
115-
);
116-
console.log('\n' + '-'.repeat(155));
106+
].join('');
107+
console.log();
108+
console.log('-'.repeat(235));
109+
console.log(line);
110+
console.log('-'.repeat(235));
117111
};
118112

119113
/**
@@ -134,11 +128,12 @@ export async function runner({ iterations, setup, name, run, skip }) {
134128
const BSONLibs = await getLibs();
135129
const setupResult = setup?.(BSONLibs) ?? null;
136130

137-
console.log('-'.repeat(155));
131+
console.log('-'.repeat(235));
138132

139133
for (const bson of BSONLibs) {
140-
const profileName = `${bson.name}_${name}`;
134+
const profileName = `${bson.name}_${name.replaceAll(' ', '-')}`;
141135
v8Profiler.startProfiling(profileName, true);
136+
v8Profiler.setGenerateType(1);
142137
const { histogram, thrownError } = await testPerformance(bson, [run, setupResult], iterations);
143138
if (thrownError != null) {
144139
console.log(

0 commit comments

Comments
 (0)