Skip to content

Commit 76dc2db

Browse files
cgeweckecgewecke
committed
Generate mocha JSON output with --matrix
1 parent 308f32d commit 76dc2db

File tree

10 files changed

+219
-15
lines changed

10 files changed

+219
-15
lines changed

‎README.md

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,8 @@ module.exports = {
126126
| measureFunctionCoverage | *boolean* | `true` | Computes function coverage. [More...][34] |
127127
| measureModifierCoverage | *boolean* | `true` | Computes each modifier invocation as a code branch. [More...][34] |
128128
| modifierWhitelist | *String[]* | `[]` | List of modifier names (ex: "onlyOwner") to exclude from branch measurement. (Useful for modifiers which prepare something instead of acting as a gate.)) |
129-
| matrixOutputPath | *String* | `./testMatrix.json` | Relative path to write test matrix JSON object to. [More...][38] |
129+
| matrixOutputPath | *String* | `./testMatrix.json` | Relative path to write test matrix JSON object to. [More...][38]|
130+
| mochaJsonOutputPath | *String* | `./mochaOutput.json` | Relative path to write mocha JSON reporter object to. [More...][38]|
130131
| abiOutputPath | *String* | `./humanReadableAbis.json` | Relative path to write diff-able ABI data to. [More...][38] |
131132
| istanbulFolder | *String* | `./coverage` | Folder location for Istanbul coverage reports. |
132133
| istanbulReporter | *Array* | `['html', 'lcov', 'text', 'json']` | [Istanbul coverage reporters][2] |

‎docs/advanced.md

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,15 @@ to guess where bugs might exist in a given codebase.
109109
Running the coverage command with `--matrix` will write [a JSON test matrix][25] which maps greppable
110110
test names to each line of code to a file named `testMatrix.json` in your project's root.
111111

112+
It also generates a `mochaOutput.json` file which contains test run data similar to that
113+
generated by mocha's built-in [JSON reporter][27].
114+
115+
In combination these data sets can be passed to Joram's Honig's [tarantula][29] tool which uses
116+
a fault localization algorithm to generate 'suspiciousness' ratings for each line of
117+
Solidity code in your project.
118+
112119
[22]: https://github.com/JoranHonig/vertigo#vertigo
113120
[23]: http://spideruci.org/papers/jones05.pdf
114121
[25]: https://github.com/sc-forks/solidity-coverage/blob/master/docs/matrix.md
122+
[27]: https://mochajs.org/api/reporters_json.js.html
123+
[29]: https://github.com/JoranHonig/tarantula

‎lib/api.js

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ class API {
3535
this.cwd = config.cwd || process.cwd();
3636
this.abiOutputPath = config.abiOutputPath || "humanReadableAbis.json";
3737
this.matrixOutputPath = config.matrixOutputPath || "testMatrix.json";
38+
this.mochaJsonOutputPath = config.mochaJsonOutputPath || "mochaOutput.json";
3839
this.matrixReporterPath = config.matrixReporterPath || "solidity-coverage/plugins/resources/matrix.js"
3940

4041
this.defaultHook = () => {};
@@ -360,6 +361,11 @@ class API {
360361
fs.writeFileSync(matrixPath, JSON.stringify(mapping, null, ' '));
361362
}
362363

364+
saveMochaJsonOutput(data){
365+
const outputPath = path.join(this.cwd, this.mochaJsonOutputPath);
366+
fs.writeFileSync(outputPath, JSON.stringify(data, null, ' '));
367+
}
368+
363369
saveHumanReadableAbis(data){
364370
const abiPath = path.join(this.cwd, this.abiOutputPath);
365371
fs.writeFileSync(abiPath, JSON.stringify(data, null, ' '));

‎plugins/resources/matrix.js

Lines changed: 79 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
const mocha = require("mocha");
22
const inherits = require("util").inherits;
33
const Spec = mocha.reporters.Spec;
4-
4+
const util = require('util')
55

66
/**
77
* This file adapted from mocha's stats-collector
@@ -40,10 +40,13 @@ function mochaStats(runner) {
4040
}
4141

4242
/**
43-
* Based on the Mocha 'Spec' reporter. Watches an Ethereum test suite run
44-
* and collects data about which tests hit which lines of code.
45-
* This "test matrix" can be used as an input to
43+
* Based on the Mocha 'Spec' reporter.
44+
*
45+
* Watches an Ethereum test suite run and collects data about which tests hit
46+
* which lines of code. This "test matrix" can be used as an input to fault localization tools
47+
* like: https://github.com/JoranHonig/tarantula
4648
*
49+
* Mocha's JSON reporter output is also generated and saved to a separate file
4750
*
4851
* @param {Object} runner mocha's runner
4952
* @param {Object} options reporter.options (see README example usage)
@@ -52,6 +55,12 @@ function Matrix(runner, options) {
5255
// Spec reporter
5356
Spec.call(this, runner, options);
5457

58+
const self = this;
59+
const tests = [];
60+
const pending = [];
61+
const failures = [];
62+
const passes = [];
63+
5564
// Initialize stats for Mocha 6+ epilogue
5665
if (!runner.stats) {
5766
mochaStats(runner);
@@ -60,7 +69,73 @@ function Matrix(runner, options) {
6069

6170
runner.on("test end", (info) => {
6271
options.reporterOptions.collectTestMatrixData(info);
72+
tests.push(info);
73+
});
74+
75+
runner.on('pass', function(info) {
76+
passes.push(info)
77+
})
78+
runner.on('fail', function(info) {
79+
failures.push(info)
80+
});
81+
82+
runner.once('end', function() {
83+
delete self.stats.start;
84+
delete self.stats.end;
85+
delete self.stats.duration;
86+
87+
var obj = {
88+
stats: self.stats,
89+
tests: tests.map(clean),
90+
failures: failures.map(clean),
91+
passes: passes.map(clean)
92+
};
93+
runner.testResults = obj;
94+
options.reporterOptions.saveMochaJsonOutput(obj)
6395
});
96+
97+
// >>>>>>>>>>>>>>>>>>>>>>>>>
98+
// Mocha JSON Reporter Utils
99+
// Code taken from:
100+
// https://mochajs.org/api/reporters_json.js.html
101+
// >>>>>>>>>>>>>>>>>>>>>>>>>
102+
function clean(info) {
103+
var err = info.err || {};
104+
if (err instanceof Error) {
105+
err = errorJSON(err);
106+
}
107+
return {
108+
title: info.title,
109+
fullTitle: info.fullTitle(),
110+
file: info.file,
111+
currentRetry: info.currentRetry(),
112+
err: cleanCycles(err)
113+
};
114+
}
115+
116+
function cleanCycles(obj) {
117+
var cache = [];
118+
return JSON.parse(
119+
JSON.stringify(obj, function(key, value) {
120+
if (typeof value === 'object' && value !== null) {
121+
if (cache.indexOf(value) !== -1) {
122+
// Instead of going in a circle, we'll print [object Object]
123+
return '' + value;
124+
}
125+
cache.push(value);
126+
}
127+
return value;
128+
})
129+
);
130+
}
131+
132+
function errorJSON(err) {
133+
var res = {};
134+
Object.getOwnPropertyNames(err).forEach(function(key) {
135+
res[key] = err[key];
136+
}, err);
137+
return res;
138+
}
64139
}
65140

66141
/**

‎plugins/resources/nomiclabs.utils.js

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -174,7 +174,8 @@ function collectTestMatrixData(args, env, api){
174174
mochaConfig = env.config.mocha || {};
175175
mochaConfig.reporter = api.matrixReporterPath;
176176
mochaConfig.reporterOptions = {
177-
collectTestMatrixData: api.collectTestMatrixData.bind(api)
177+
collectTestMatrixData: api.collectTestMatrixData.bind(api),
178+
saveMochaJsonOutput: api.saveMochaJsonOutput.bind(api)
178179
}
179180
env.config.mocha = mochaConfig;
180181
}

‎plugins/resources/truffle.utils.js

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -240,7 +240,8 @@ function collectTestMatrixData(config, api){
240240
config.mocha = config.mocha || {};
241241
config.mocha.reporter = api.matrixReporterPath;
242242
config.mocha.reporterOptions = {
243-
collectTestMatrixData: api.collectTestMatrixData.bind(api)
243+
collectTestMatrixData: api.collectTestMatrixData.bind(api),
244+
saveMochaJsonOutput: api.saveMochaJsonOutput.bind(api)
244245
}
245246
}
246247
}

‎test/integration/projects/matrix/.solcover.js

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ module.exports = {
99
// "solidity-coverage/plugins/resources/matrix.js"
1010
matrixReporterPath: reporterPath,
1111
matrixOutputPath: "alternateTestMatrix.json",
12+
mochaJsonOutputPath: "alternateMochaOutput.json",
1213

1314
skipFiles: ['Migrations.sol'],
1415
silent: process.env.SILENT ? true : false,

‎test/integration/projects/matrix/expectedMochaOutput.json

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
{
2+
"stats": {
3+
"suites": 2,
4+
"tests": 6,
5+
"passes": 6,
6+
"pending": 0,
7+
"failures": 0
8+
},
9+
"tests": [
10+
{
11+
"title": "sends to A",
12+
"fullTitle": "Contract: Matrix A and B sends to A",
13+
"file": "/Users/cgewecke/code/sc-forks/solidity-coverage/sc_temp/test/matrix_a_b.js",
14+
"currentRetry": 0,
15+
"err": {}
16+
},
17+
{
18+
"title": "sends to A",
19+
"fullTitle": "Contract: Matrix A and B sends to A",
20+
"file": "/Users/cgewecke/code/sc-forks/solidity-coverage/sc_temp/test/matrix_a_b.js",
21+
"currentRetry": 0,
22+
"err": {}
23+
},
24+
{
25+
"title": "calls B",
26+
"fullTitle": "Contract: Matrix A and B calls B",
27+
"file": "/Users/cgewecke/code/sc-forks/solidity-coverage/sc_temp/test/matrix_a_b.js",
28+
"currentRetry": 0,
29+
"err": {}
30+
},
31+
{
32+
"title": "sends to B",
33+
"fullTitle": "Contract: Matrix A and B sends to B",
34+
"file": "/Users/cgewecke/code/sc-forks/solidity-coverage/sc_temp/test/matrix_a_b.js",
35+
"currentRetry": 0,
36+
"err": {}
37+
},
38+
{
39+
"title": "sends",
40+
"fullTitle": "Contract: MatrixA sends",
41+
"file": "/Users/cgewecke/code/sc-forks/solidity-coverage/sc_temp/test/matrix_a.js",
42+
"currentRetry": 0,
43+
"err": {}
44+
},
45+
{
46+
"title": "calls",
47+
"fullTitle": "Contract: MatrixA calls",
48+
"file": "/Users/cgewecke/code/sc-forks/solidity-coverage/sc_temp/test/matrix_a.js",
49+
"currentRetry": 0,
50+
"err": {}
51+
}
52+
],
53+
"failures": [],
54+
"passes": [
55+
{
56+
"title": "sends to A",
57+
"fullTitle": "Contract: Matrix A and B sends to A",
58+
"file": "/Users/cgewecke/code/sc-forks/solidity-coverage/sc_temp/test/matrix_a_b.js",
59+
"currentRetry": 0,
60+
"err": {}
61+
},
62+
{
63+
"title": "sends to A",
64+
"fullTitle": "Contract: Matrix A and B sends to A",
65+
"file": "/Users/cgewecke/code/sc-forks/solidity-coverage/sc_temp/test/matrix_a_b.js",
66+
"currentRetry": 0,
67+
"err": {}
68+
},
69+
{
70+
"title": "calls B",
71+
"fullTitle": "Contract: Matrix A and B calls B",
72+
"file": "/Users/cgewecke/code/sc-forks/solidity-coverage/sc_temp/test/matrix_a_b.js",
73+
"currentRetry": 0,
74+
"err": {}
75+
},
76+
{
77+
"title": "sends to B",
78+
"fullTitle": "Contract: Matrix A and B sends to B",
79+
"file": "/Users/cgewecke/code/sc-forks/solidity-coverage/sc_temp/test/matrix_a_b.js",
80+
"currentRetry": 0,
81+
"err": {}
82+
},
83+
{
84+
"title": "sends",
85+
"fullTitle": "Contract: MatrixA sends",
86+
"file": "/Users/cgewecke/code/sc-forks/solidity-coverage/sc_temp/test/matrix_a.js",
87+
"currentRetry": 0,
88+
"err": {}
89+
},
90+
{
91+
"title": "calls",
92+
"fullTitle": "Contract: MatrixA calls",
93+
"file": "/Users/cgewecke/code/sc-forks/solidity-coverage/sc_temp/test/matrix_a.js",
94+
"currentRetry": 0,
95+
"err": {}
96+
}
97+
]
98+
}
99+

‎test/units/hardhat/flags.js

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -184,12 +184,18 @@ describe('Hardhat Plugin: command line options', function() {
184184
await this.env.run("coverage", taskArgs);
185185

186186
// Integration test checks output path configurabililty
187-
const altPath = path.join(process.cwd(), './alternateTestMatrix.json');
188-
const expPath = path.join(process.cwd(), './expectedTestMatrixHardhat.json');
189-
const producedMatrix = require(altPath)
190-
const expectedMatrix = require(expPath);
187+
const altMatrixPath = path.join(process.cwd(), './alternateTestMatrix.json');
188+
const expMatrixPath = path.join(process.cwd(), './expectedTestMatrixHardhat.json');
189+
const altMochaPath = path.join(process.cwd(), './alternateMochaOutput.json');
190+
const expMochaPath = path.join(process.cwd(), './expectedMochaOutput.json');
191+
192+
const producedMatrix = require(altMatrixPath)
193+
const expectedMatrix = require(expMatrixPath);
194+
const producedMochaOutput = require(altMochaPath);
195+
const expectedMochaOutput = require(expMochaPath);
191196

192197
assert.deepEqual(producedMatrix, expectedMatrix);
198+
assert.deepEqual(producedMochaOutput, expectedMochaOutput);
193199
});
194200

195201
it('--abi', async function(){

‎test/units/truffle/flags.js

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -280,13 +280,18 @@ describe('Truffle Plugin: command line options', function() {
280280
await plugin(truffleConfig);
281281

282282
// Integration test checks output path configurabililty
283-
const altPath = path.join(process.cwd(), mock.pathToTemp('./alternateTestMatrix.json'));
284-
const expPath = path.join(process.cwd(), mock.pathToTemp('./expectedTestMatrixHardhat.json'));
283+
const altMatrixPath = path.join(process.cwd(), mock.pathToTemp('./alternateTestMatrix.json'));
284+
const expMatrixPath = path.join(process.cwd(), mock.pathToTemp('./expectedTestMatrixHardhat.json'));
285+
const altMochaPath = path.join(process.cwd(), mock.pathToTemp('./alternateMochaOutput.json'));
286+
const expMochaPath = path.join(process.cwd(), mock.pathToTemp('./expectedMochaOutput.json'));
285287

286-
const producedMatrix = require(altPath)
287-
const expectedMatrix = require(expPath);
288+
const producedMatrix = require(altMatrixPath)
289+
const expectedMatrix = require(expMatrixPath);
290+
const producedMochaOutput = require(altMochaPath);
291+
const expectedMochaOutput = require(expMochaPath);
288292

289293
assert.deepEqual(producedMatrix, expectedMatrix);
294+
assert.deepEqual(producedMochaOutput, expectedMochaOutput);
290295
process.env.TRUFFLE_TEST = false;
291296
});
292297

0 commit comments

Comments
 (0)