Skip to content

Commit e9133d7

Browse files
cgeweckecgewecke
committed
Generate mocha JSON output with --matrix (#601)
1 parent adcaab5 commit e9133d7

File tree

12 files changed

+243
-30
lines changed

12 files changed

+243
-30
lines changed

‎README.md

+2-1
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,8 @@ module.exports = {
8585
| measureFunctionCoverage | *boolean* | `true` | Computes function coverage. [More...][34] |
8686
| measureModifierCoverage | *boolean* | `true` | Computes each modifier invocation as a code branch. [More...][34] |
8787
| modifierWhitelist | *String[]* | `[]` | List of modifier names (ex: "onlyOwner") to exclude from branch measurement. (Useful for modifiers which prepare something instead of acting as a gate.)) |
88-
| matrixOutputPath | *String* | `./testMatrix.json` | Relative path to write test matrix JSON object to. [More...][39] |
88+
| matrixOutputPath | *String* | `./testMatrix.json` | Relative path to write test matrix JSON object to. [More...][39]|
89+
| mochaJsonOutputPath | *String* | `./mochaOutput.json` | Relative path to write mocha JSON reporter object to. [More...][39]|
8990
| abiOutputPath | *String* | `./humanReadableAbis.json` | Relative path to write diff-able ABI data to |
9091
| istanbulFolder | *String* | `./coverage` | Folder location for Istanbul coverage reports. |
9192
| istanbulReporter | *Array* | `['html', 'lcov', 'text', 'json']` | [Istanbul coverage reporters][2] |

‎docs/advanced.md

+9
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,15 @@ to guess where bugs might exist in a given codebase.
118118
Running the coverage command with `--matrix` will write [a JSON test matrix][25] which maps greppable
119119
test names to each line of code to a file named `testMatrix.json` in your project's root.
120120

121+
It also generates a `mochaOutput.json` file which contains test run data similar to that
122+
generated by mocha's built-in [JSON reporter][27].
123+
124+
In combination these data sets can be passed to Joram's Honig's [tarantula][29] tool which uses
125+
a fault localization algorithm to generate 'suspiciousness' ratings for each line of
126+
Solidity code in your project.
127+
121128
[22]: https://github.com/JoranHonig/vertigo#vertigo
122129
[23]: http://spideruci.org/papers/jones05.pdf
123130
[25]: https://github.com/sc-forks/solidity-coverage/blob/master/docs/matrix.md
131+
[27]: https://mochajs.org/api/reporters_json.js.html
132+
[29]: https://github.com/JoranHonig/tarantula

‎lib/api.js

+21-13
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ class API {
3535
this.cwd = config.cwd || process.cwd();
3636
this.abiOutputPath = config.abiOutputPath || "humanReadableAbis.json";
3737
this.matrixOutputPath = config.matrixOutputPath || "testMatrix.json";
38+
this.mochaJsonOutputPath = config.mochaJsonOutputPath || "mochaOutput.json";
3839
this.matrixReporterPath = config.matrixReporterPath || "solidity-coverage/plugins/resources/matrix.js"
3940

4041
this.defaultHook = () => {};
@@ -321,29 +322,31 @@ class API {
321322
id
322323
} = this.instrumenter.instrumentationData[hash];
323324

324-
if (type === 'line' && hits > 0){
325+
if (type === 'line'){
325326
if (!this.testMatrix[contractPath]){
326327
this.testMatrix[contractPath] = {};
327328
}
328329
if (!this.testMatrix[contractPath][id]){
329330
this.testMatrix[contractPath][id] = [];
330331
}
331332

332-
// Search for and exclude duplicate entries
333-
let duplicate = false;
334-
for (const item of this.testMatrix[contractPath][id]){
335-
if (item.title === title && item.file === file){
336-
duplicate = true;
337-
break;
333+
if (hits > 0){
334+
// Search for and exclude duplicate entries
335+
let duplicate = false;
336+
for (const item of this.testMatrix[contractPath][id]){
337+
if (item.title === title && item.file === file){
338+
duplicate = true;
339+
break;
340+
}
338341
}
339-
}
340342

341-
if (!duplicate) {
342-
this.testMatrix[contractPath][id].push({title, file});
343-
}
343+
if (!duplicate) {
344+
this.testMatrix[contractPath][id].push({title, file});
345+
}
344346

345-
// Reset line data
346-
this.instrumenter.instrumentationData[hash].hits = 0;
347+
// Reset line data
348+
this.instrumenter.instrumentationData[hash].hits = 0;
349+
}
347350
}
348351
}
349352
}
@@ -363,6 +366,11 @@ class API {
363366
fs.writeFileSync(matrixPath, JSON.stringify(mapping, null, ' '));
364367
}
365368

369+
saveMochaJsonOutput(data){
370+
const outputPath = path.join(this.cwd, this.mochaJsonOutputPath);
371+
fs.writeFileSync(outputPath, JSON.stringify(data, null, ' '));
372+
}
373+
366374
saveHumanReadableAbis(data){
367375
const abiPath = path.join(this.cwd, this.abiOutputPath);
368376
fs.writeFileSync(abiPath, JSON.stringify(data, null, ' '));

‎plugins/resources/matrix.js

+78-4
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
const mocha = require("mocha");
22
const inherits = require("util").inherits;
33
const Spec = mocha.reporters.Spec;
4-
4+
const path = require('path');
55

66
/**
77
* This file adapted from mocha's stats-collector
@@ -40,10 +40,13 @@ function mochaStats(runner) {
4040
}
4141

4242
/**
43-
* Based on the Mocha 'Spec' reporter. Watches an Ethereum test suite run
44-
* and collects data about which tests hit which lines of code.
45-
* This "test matrix" can be used as an input to
43+
* Based on the Mocha 'Spec' reporter.
44+
*
45+
* Watches an Ethereum test suite run and collects data about which tests hit
46+
* which lines of code. This "test matrix" can be used as an input to fault localization tools
47+
* like: https://github.com/JoranHonig/tarantula
4648
*
49+
* Mocha's JSON reporter output is also generated and saved to a separate file
4750
*
4851
* @param {Object} runner mocha's runner
4952
* @param {Object} options reporter.options (see README example usage)
@@ -52,6 +55,11 @@ function Matrix(runner, options) {
5255
// Spec reporter
5356
Spec.call(this, runner, options);
5457

58+
const self = this;
59+
const tests = [];
60+
const failures = [];
61+
const passes = [];
62+
5563
// Initialize stats for Mocha 6+ epilogue
5664
if (!runner.stats) {
5765
mochaStats(runner);
@@ -60,7 +68,73 @@ function Matrix(runner, options) {
6068

6169
runner.on("test end", (info) => {
6270
options.reporterOptions.collectTestMatrixData(info);
71+
tests.push(info);
72+
});
73+
74+
runner.on('pass', function(info) {
75+
passes.push(info)
76+
})
77+
runner.on('fail', function(info) {
78+
failures.push(info)
79+
});
80+
81+
runner.once('end', function() {
82+
delete self.stats.start;
83+
delete self.stats.end;
84+
delete self.stats.duration;
85+
86+
var obj = {
87+
stats: self.stats,
88+
tests: tests.map(clean),
89+
failures: failures.map(clean),
90+
passes: passes.map(clean)
91+
};
92+
runner.testResults = obj;
93+
options.reporterOptions.saveMochaJsonOutput(obj)
6394
});
95+
96+
// >>>>>>>>>>>>>>>>>>>>>>>>>
97+
// Mocha JSON Reporter Utils
98+
// Code taken from:
99+
// https://mochajs.org/api/reporters_json.js.html
100+
// >>>>>>>>>>>>>>>>>>>>>>>>>
101+
function clean(info) {
102+
var err = info.err || {};
103+
if (err instanceof Error) {
104+
err = errorJSON(err);
105+
}
106+
return {
107+
title: info.title,
108+
fullTitle: info.fullTitle(),
109+
file: path.relative(options.reporterOptions.cwd, info.file),
110+
currentRetry: info.currentRetry(),
111+
err: cleanCycles(err)
112+
};
113+
}
114+
115+
function cleanCycles(obj) {
116+
var cache = [];
117+
return JSON.parse(
118+
JSON.stringify(obj, function(key, value) {
119+
if (typeof value === 'object' && value !== null) {
120+
if (cache.indexOf(value) !== -1) {
121+
// Instead of going in a circle, we'll print [object Object]
122+
return '' + value;
123+
}
124+
cache.push(value);
125+
}
126+
return value;
127+
})
128+
);
129+
}
130+
131+
function errorJSON(err) {
132+
var res = {};
133+
Object.getOwnPropertyNames(err).forEach(function(key) {
134+
res[key] = err[key];
135+
}, err);
136+
return res;
137+
}
64138
}
65139

66140
/**

‎plugins/resources/nomiclabs.utils.js

+3-1
Original file line numberDiff line numberDiff line change
@@ -175,7 +175,9 @@ function collectTestMatrixData(args, env, api){
175175
mochaConfig = env.config.mocha || {};
176176
mochaConfig.reporter = api.matrixReporterPath;
177177
mochaConfig.reporterOptions = {
178-
collectTestMatrixData: api.collectTestMatrixData.bind(api)
178+
collectTestMatrixData: api.collectTestMatrixData.bind(api),
179+
saveMochaJsonOutput: api.saveMochaJsonOutput.bind(api),
180+
cwd: api.cwd
179181
}
180182
env.config.mocha = mochaConfig;
181183
}

‎plugins/resources/truffle.utils.js

+3-1
Original file line numberDiff line numberDiff line change
@@ -238,7 +238,9 @@ function collectTestMatrixData(config, api){
238238
config.mocha = config.mocha || {};
239239
config.mocha.reporter = api.matrixReporterPath;
240240
config.mocha.reporterOptions = {
241-
collectTestMatrixData: api.collectTestMatrixData.bind(api)
241+
collectTestMatrixData: api.collectTestMatrixData.bind(api),
242+
saveMochaJsonOutput: api.saveMochaJsonOutput.bind(api),
243+
cwd: api.cwd
242244
}
243245
}
244246
}

‎test/integration/projects/matrix/.solcover.js

+1
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ module.exports = {
99
// "solidity-coverage/plugins/resources/matrix.js"
1010
matrixReporterPath: reporterPath,
1111
matrixOutputPath: "alternateTestMatrix.json",
12+
mochaJsonOutputPath: "alternateMochaOutput.json",
1213

1314
skipFiles: ['Migrations.sol'],
1415
silent: process.env.SILENT ? true : false,

‎test/integration/projects/matrix/contracts/MatrixA.sol

+4
Original file line numberDiff line numberDiff line change
@@ -14,4 +14,8 @@ contract MatrixA {
1414
uint y = 5;
1515
return y;
1616
}
17+
18+
function unhit() public {
19+
uint z = 7;
20+
}
1721
}

‎test/integration/projects/matrix/expectedMochaOutput.json

+99
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
{
2+
"stats": {
3+
"suites": 2,
4+
"tests": 6,
5+
"passes": 6,
6+
"pending": 0,
7+
"failures": 0
8+
},
9+
"tests": [
10+
{
11+
"title": "sends to A",
12+
"fullTitle": "Contract: Matrix A and B sends to A",
13+
"file": "test/matrix_a_b.js",
14+
"currentRetry": 0,
15+
"err": {}
16+
},
17+
{
18+
"title": "sends to A",
19+
"fullTitle": "Contract: Matrix A and B sends to A",
20+
"file": "test/matrix_a_b.js",
21+
"currentRetry": 0,
22+
"err": {}
23+
},
24+
{
25+
"title": "calls B",
26+
"fullTitle": "Contract: Matrix A and B calls B",
27+
"file": "test/matrix_a_b.js",
28+
"currentRetry": 0,
29+
"err": {}
30+
},
31+
{
32+
"title": "sends to B",
33+
"fullTitle": "Contract: Matrix A and B sends to B",
34+
"file": "test/matrix_a_b.js",
35+
"currentRetry": 0,
36+
"err": {}
37+
},
38+
{
39+
"title": "sends",
40+
"fullTitle": "Contract: MatrixA sends",
41+
"file": "test/matrix_a.js",
42+
"currentRetry": 0,
43+
"err": {}
44+
},
45+
{
46+
"title": "calls",
47+
"fullTitle": "Contract: MatrixA calls",
48+
"file": "test/matrix_a.js",
49+
"currentRetry": 0,
50+
"err": {}
51+
}
52+
],
53+
"failures": [],
54+
"passes": [
55+
{
56+
"title": "sends to A",
57+
"fullTitle": "Contract: Matrix A and B sends to A",
58+
"file": "test/matrix_a_b.js",
59+
"currentRetry": 0,
60+
"err": {}
61+
},
62+
{
63+
"title": "sends to A",
64+
"fullTitle": "Contract: Matrix A and B sends to A",
65+
"file": "test/matrix_a_b.js",
66+
"currentRetry": 0,
67+
"err": {}
68+
},
69+
{
70+
"title": "calls B",
71+
"fullTitle": "Contract: Matrix A and B calls B",
72+
"file": "test/matrix_a_b.js",
73+
"currentRetry": 0,
74+
"err": {}
75+
},
76+
{
77+
"title": "sends to B",
78+
"fullTitle": "Contract: Matrix A and B sends to B",
79+
"file": "test/matrix_a_b.js",
80+
"currentRetry": 0,
81+
"err": {}
82+
},
83+
{
84+
"title": "sends",
85+
"fullTitle": "Contract: MatrixA sends",
86+
"file": "test/matrix_a.js",
87+
"currentRetry": 0,
88+
"err": {}
89+
},
90+
{
91+
"title": "calls",
92+
"fullTitle": "Contract: MatrixA calls",
93+
"file": "test/matrix_a.js",
94+
"currentRetry": 0,
95+
"err": {}
96+
}
97+
]
98+
}
99+

‎test/integration/projects/matrix/expectedTestMatrixHardhat.json

+4-2
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,8 @@
2121
"title": "calls",
2222
"file": "test/matrix_a.js"
2323
}
24-
]
24+
],
25+
"19": []
2526
},
2627
"contracts/MatrixB.sol": {
2728
"10": [
@@ -43,4 +44,5 @@
4344
}
4445
]
4546
}
46-
}
47+
}
48+

‎test/units/hardhat/flags.js

+10-4
Original file line numberDiff line numberDiff line change
@@ -184,12 +184,18 @@ describe('Hardhat Plugin: command line options', function() {
184184
await this.env.run("coverage", taskArgs);
185185

186186
// Integration test checks output path configurabililty
187-
const altPath = path.join(process.cwd(), './alternateTestMatrix.json');
188-
const expPath = path.join(process.cwd(), './expectedTestMatrixHardhat.json');
189-
const producedMatrix = require(altPath)
190-
const expectedMatrix = require(expPath);
187+
const altMatrixPath = path.join(process.cwd(), './alternateTestMatrix.json');
188+
const expMatrixPath = path.join(process.cwd(), './expectedTestMatrixHardhat.json');
189+
const altMochaPath = path.join(process.cwd(), './alternateMochaOutput.json');
190+
const expMochaPath = path.join(process.cwd(), './expectedMochaOutput.json');
191+
192+
const producedMatrix = require(altMatrixPath)
193+
const expectedMatrix = require(expMatrixPath);
194+
const producedMochaOutput = require(altMochaPath);
195+
const expectedMochaOutput = require(expMochaPath);
191196

192197
assert.deepEqual(producedMatrix, expectedMatrix);
198+
assert.deepEqual(producedMochaOutput, expectedMochaOutput);
193199
});
194200

195201
it('--abi', async function(){

0 commit comments

Comments
 (0)