Skip to content

♻️ Refactor reporter code #31

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tolauwae merged 4 commits into from
Dec 12, 2024
Merged

♻️ Refactor reporter code #31

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
edffed5
Add example test
tolauwae Dec 10, 2024
3de3672
Refactor reporter code
tolauwae Dec 11, 2024
9fb4d65
Fix verbosity levels again
tolauwae Dec 12, 2024
f5a3d15
Fix eslint warnings
tolauwae Dec 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions .eslint.config.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,17 @@ import stylisticJs from '@stylistic/eslint-plugin-js'

/** @type {import('eslint').Linter.Config[]} */
export default [
pluginJs.configs.recommended,
...tseslint.configs.recommended,
{ plugins: {
'@stylistic/js': stylisticJs
}, rules: {
'@stylistic/js/indent': ['error', 4],
'@typescript-eslint/no-wrapper-object-types': 'off'
}
},
{files: ["**/*.{js,mjs,cjs,ts}"]},
{files: ["**/*.js"], languageOptions: {sourceType: "commonjs"}},
{languageOptions: { globals: globals.browser }},
pluginJs.configs.recommended,
...tseslint.configs.recommended,

];
1 change: 1 addition & 0 deletions package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
"test:prebuild": "npm run build && npm run build:tests",
"test:all": "npm run test:prebuild && npm run test:ava",
"test:ava": "ava",
"test:example": "npx ts-node ./tests/examples/example.ts",
"coverage:test:ava": "c8 --src src/ --all ava"
},
"dependencies": {
Expand Down
17 changes: 9 additions & 8 deletions src/framework/Framework.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,11 @@ import {HybridScheduler, Scheduler} from './Scheduler';
import {TestScenario} from './scenario/TestScenario';

import {TestbedSpecification} from '../testbeds/TestbedSpecification';
import {Reporter, SuiteResults} from '../reporter/Reporter';

import {StyleType} from '../reporter';
import {styling} from '../reporter/Style';
import {SuiteResult} from '../reporter/Results';
import {Reporter} from '../reporter/Reporter';

export interface Suite {

Expand Down Expand Up @@ -90,10 +91,10 @@ export class Framework {
for (const suite of suites) {
for (const testee of suite.testees) {
const order: TestScenario[] = suite.scheduler.sequential(suite);
const result: SuiteResults = new SuiteResults(suite, testee);
const result: SuiteResult = new SuiteResult(suite);

const first: TestScenario = order[0];
await timeout<Object | void>('Initialize testbed', testee.connector.timeout, testee.initialize(first.program, first.args ?? []).catch((e: Error) => result.error = e));
await timeout<Object | void>('Initialize testbed', testee.connector.timeout, testee.initialize(first.program, first.args ?? []).catch((e: Error) => result.error(e.message)));

await this.runSuite(result, testee, order);
this.reporter.report(result);
Expand All @@ -114,10 +115,10 @@ export class Framework {
await Promise.all(suites.map(async (suite: Suite) => {
await Promise.all(suite.testees.map(async (testee: Testee) => {
const order: TestScenario[] = suite.scheduler.sequential(suite);
const result: SuiteResults = new SuiteResults(suite, testee);
const result: SuiteResult = new SuiteResult(suite);

const first: TestScenario = order[0];
await timeout<Object | void>('Initialize testbed', testee.connector.timeout, testee.initialize(first.program, first.args ?? []).catch((e: Error) => result.error = e));
await timeout<Object | void>('Initialize testbed', testee.connector.timeout, testee.initialize(first.program, first.args ?? []).catch((e: Error) => result.error(e.message)));

await this.runSuite(result, testee, order);
this.reporter.report(result);
Expand All @@ -139,10 +140,10 @@ export class Framework {
const order: TestScenario[][] = suite.scheduler.parallel(suite, suite.testees.length);
await Promise.all(suite.testees.map(async (testee: Testee, i: number) => {
// console.log(`scheduling on ${testee.name}`)
const result: SuiteResults = new SuiteResults(suite, testee);
const result: SuiteResult = new SuiteResult(suite);

const first: TestScenario = order[i][0];
await timeout<Object | void>('Initialize testbed', testee.connector.timeout, testee.initialize(first.program, first.args ?? []).catch((e: Error) => result.error = e));
await timeout<Object | void>('Initialize testbed', testee.connector.timeout, testee.initialize(first.program, first.args ?? []).catch((e: Error) => result.error(e.message)));

for (let j = i; j < order.length; j += suite.testees.length) {
await this.runSuite(result, testee, order[j]);
Expand All @@ -159,7 +160,7 @@ export class Framework {
this.reporter.results(t1 - t0);
}

private async runSuite(result: SuiteResults, testee: Testee, order: TestScenario[]) {
private async runSuite(result: SuiteResult, testee: Testee, order: TestScenario[]) {
for (const test of order) {
await testee.describe(test, result, this.runs);
}
Expand Down
79 changes: 39 additions & 40 deletions src/framework/Testee.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,11 @@ import {TestScenario} from './scenario/TestScenario';
import {OutofPlaceSpecification, PlatformType, TestbedSpecification} from '../testbeds/TestbedSpecification';
import {CompileOutput, CompilerFactory} from '../manage/Compiler';
import {WABT} from '../util/env';
import {Completion, expect, ScenarioResult, SuiteResults} from '../reporter/Reporter';
import {Outcome} from '../reporter/describers/Describer';
import {WASM} from '../sourcemap/Wasm';
import {DummyProxy} from '../testbeds/Emulator';
import {Result} from '../reporter/Result';
import {ScenarioResult, Skipped, StepOutcome, SuiteResult} from '../reporter/Results';
import {Verifier} from './Verifier';

export function timeout<T>(label: string, time: number, promise: Promise<T>): Promise<T> {
if (time === 0) {
Expand Down Expand Up @@ -46,14 +47,14 @@ export function getValue(object: any, field: string): any {
}

export enum Target {
supervisor,
proxy
supervisor = 'supervisor',
proxy = 'proxy'
}

export class Testee { // TODO unified with testbed interface

/** The current state for each described test */
private states: Map<string, Result> = new Map<string, Result>();
private states: Map<string, StepOutcome> = new Map<string, StepOutcome>();

/** Factory to establish new connections to VMs */
public readonly connector: TestbedFactory;
Expand Down Expand Up @@ -132,16 +133,16 @@ export class Testee { // TODO unified with testbed interface
}

private run(name: string, limit: number, fn: () => Promise<any>) {
return timeout<Object | void>(name, limit, fn());
return timeout<object | void>(name, limit, fn());
}

private step(name: string, limit: number, fn: () => Promise<any>) {
return timeout<Object | void>(name, limit, fn());
return timeout<object | void>(name, limit, fn());
}

public async describe(description: TestScenario, suiteResult: SuiteResults, runs: number = 1) {
public async describe(description: TestScenario, suiteResult: SuiteResult, runs: number = 1) {
const testee = this;
const scenarioResult: ScenarioResult = new ScenarioResult(description, testee);
const scenarioResult: ScenarioResult = new ScenarioResult(description);

if (description.skip) {
return;
Expand All @@ -154,11 +155,11 @@ export class Testee { // TODO unified with testbed interface
await this.run('Check for failing dependencies', testee.timeout, async function () {
const failedDependencies: TestScenario[] = testee.failedDependencies(description);
if (failedDependencies.length > 0) {
testee.states.set(description.title, new Result('Skipping', 'Test has failing dependencies', Completion.skipped));
testee.states.set(description.title, new Skipped('Skipping', 'Test has failing dependencies'));
throw new Error(`Skipped: failed dependent tests: ${failedDependencies.map(dependence => dependence.title)}`);
}
}).catch((e: Error) => {
scenarioResult.error = e;
scenarioResult.error(e.message);
});

await this.run('Compile and upload program', testee.connector.timeout, async function () {
Expand All @@ -169,31 +170,31 @@ export class Testee { // TODO unified with testbed interface

const compiled: CompileOutput = await new CompilerFactory(WABT).pickCompiler(description.program).compile(description.program);
try {
await timeout<Object | void>(`uploading module`, testee.timeout, testee.bed()!.sendRequest(new SourceMap.Mapping(), Message.updateModule(compiled.file))).catch((e) => Promise.reject(e));
await timeout<object | void>(`uploading module`, testee.timeout, testee.bed()!.sendRequest(new SourceMap.Mapping(), Message.updateModule(compiled.file))).catch((e) => Promise.reject(e));
testee.current = description.program;
} catch (e) {
await testee.initialize(description.program, description.args ?? []).catch((o) => Promise.reject(o));
}
}).catch((e: Error | string) => {
if (typeof e === 'string') {
scenarioResult.error = new Error(e);
scenarioResult.error(e);
} else {
scenarioResult.error = e;
scenarioResult.error(e.toString());
}
});

await this.run('Get source mapping', testee.connector.timeout, async function () {
map = await testee.mapper.map(description.program);
}).catch((e: Error | string) => {
if (typeof e === 'string') {
scenarioResult.error = new Error(e);
scenarioResult.error(e);
} else {
scenarioResult.error = e;
scenarioResult.error(e.toString());
}
});

if (scenarioResult.error) {
suiteResult.scenarios.push(scenarioResult);
if (scenarioResult.outcome === Outcome.error) {
suiteResult.add(scenarioResult);
return;
}

Expand All @@ -205,62 +206,60 @@ export class Testee { // TODO unified with testbed interface
await this.run('resetting before retry', testee.timeout, async function () {
await testee.reset(testee.testbed);
}).catch((e: Error) => {
scenarioResult.error = e;
scenarioResult.error(e.toString());
});
}

for (const step of description.steps ?? []) {
/** Perform the step and check if expectations were met */
await this.step(step.title, testee.timeout, async function () {
let result: Result = new Result(step.title, 'incomplete');
const verifier: Verifier = new Verifier(step);
if (testee.bed(step.target ?? Target.supervisor) === undefined) {
testee.states.set(description.title, result);
result.error('Cannot run test: no debugger connection.');
testee.states.set(description.title, result);
testee.states.set(description.title, verifier.error('Cannot run test: no debugger connection.'));
return;
}

let actual: Object | void;
let actual: object | void;
if (step.instruction.kind === Kind.Action) {
actual = await timeout<Object | void>(`performing action . ${step.title}`, testee.timeout,
actual = await timeout<object | void>(`performing action . ${step.title}`, testee.timeout,
step.instruction.value.act(testee)).catch((err) => {
result.error(err);
testee.states.set(description.title, verifier.error(err));
return;
});
} else {
actual = await testee.recoverable(testee, step.instruction.value, map,
(testee, req, map) => timeout<Object | void>(`sending instruction ${req.type}`, testee.timeout,
(testee, req, map) => timeout<object | void>(`sending instruction ${req.type}`, testee.timeout,
testee.bed(step.target ?? Target.supervisor)!.sendRequest(map, req)),
(testee) => testee.run(`Recover: re-initialize ${testee.testbed?.name}`, testee.connector.timeout, async function () {
await testee.initialize(description.program, description.args ?? []).catch((o) => {
return Promise.reject(o)
});
}), 1).catch((e: Error) => {
result.completion = (e.message.includes('timeout')) ? Completion.timedout : Completion.error;
result.description = e.message;
const result = new StepOutcome(step);
testee.states.set(description.title, result.update((e.message.includes('timeout')) ? Outcome.timedout : Outcome.error, e.message));
});
}

if (result.completion === Completion.uncommenced) {
result = expect(step, actual, previous);
}
const result = verifier.verify(actual, previous);

if (actual !== undefined) {
previous = actual;
}

testee.states.set(description.title, result);
scenarioResult.results.push(result);
scenarioResult.add(result);
});
}
suiteResult.scenarios.push(scenarioResult);
suiteResult.add(scenarioResult);
}
}

/* eslint @typescript-eslint/no-explicit-any: off */
private async recoverable(testee: Testee, step: Request<any>, map: SourceMap.Mapping,
attempt: (t: Testee, req: Request<any>, m: SourceMap.Mapping) => Promise<Object | void>,
attempt: (t: Testee, req: Request<any>, m: SourceMap.Mapping) => Promise<object | void>,
recover: (t: Testee) => Promise<any>,
retries: number = 0): Promise<Object | void> {
let result: Object | void = undefined;
retries: number = 0): Promise<object | void> {
let result: object | void = undefined;
let error;
while (0 <= retries && result === undefined) {
result = await attempt(testee, step, map).catch(async (err) => {
Expand All @@ -281,7 +280,7 @@ export class Testee { // TODO unified with testbed interface
if (instance === undefined) {
this.framework.reporter.error('Cannot run test: no debugger connection.'); // todo
} else {
await timeout<Object | void>('resetting vm', this.timeout, this.testbed!.sendRequest(new SourceMap.Mapping(), Message.reset));
await timeout<object | void>('resetting vm', this.timeout, this.testbed!.sendRequest(new SourceMap.Mapping(), Message.reset));
}
}

Expand All @@ -293,8 +292,8 @@ export class Testee { // TODO unified with testbed interface
private failedDependencies(description: TestScenario): TestScenario[] {
return (description?.dependencies ?? []).filter(dependence => {
if (this.states.get(dependence.title)) {
const c = this.states.get(dependence.title)!.completion;
return !(c === Completion.succeeded || c === Completion.uncommenced);
const c = this.states.get(dependence.title)!.outcome;
return !(c === Outcome.succeeded || c === Outcome.uncommenced);
} else {
return false;
}
Expand Down
Loading
Loading