diff --git a/.eslint.config.mjs b/.eslint.config.mjs index a64161f..771747b 100644 --- a/.eslint.config.mjs +++ b/.eslint.config.mjs @@ -6,15 +6,17 @@ import stylisticJs from '@stylistic/eslint-plugin-js' /** @type {import('eslint').Linter.Config[]} */ export default [ + pluginJs.configs.recommended, + ...tseslint.configs.recommended, { plugins: { '@stylistic/js': stylisticJs }, rules: { '@stylistic/js/indent': ['error', 4], + '@typescript-eslint/no-wrapper-object-types': 'off' } }, {files: ["**/*.{js,mjs,cjs,ts}"]}, {files: ["**/*.js"], languageOptions: {sourceType: "commonjs"}}, {languageOptions: { globals: globals.browser }}, - pluginJs.configs.recommended, - ...tseslint.configs.recommended, + ]; diff --git a/package.json b/package.json index 664186b..4a9cdeb 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "test:prebuild": "npm run build && npm run build:tests", "test:all": "npm run test:prebuild && npm run test:ava", "test:ava": "ava", + "test:example": "npx ts-node ./tests/examples/example.ts", "coverage:test:ava": "c8 --src src/ --all ava" }, "dependencies": { diff --git a/src/framework/Framework.ts b/src/framework/Framework.ts index 83d6459..ced8ed0 100644 --- a/src/framework/Framework.ts +++ b/src/framework/Framework.ts @@ -3,10 +3,11 @@ import {HybridScheduler, Scheduler} from './Scheduler'; import {TestScenario} from './scenario/TestScenario'; import {TestbedSpecification} from '../testbeds/TestbedSpecification'; -import {Reporter, SuiteResults} from '../reporter/Reporter'; import {StyleType} from '../reporter'; import {styling} from '../reporter/Style'; +import {SuiteResult} from '../reporter/Results'; +import {Reporter} from '../reporter/Reporter'; export interface Suite { @@ -90,10 +91,10 @@ export class Framework { for (const suite of suites) { for (const testee of suite.testees) { const order: TestScenario[] = suite.scheduler.sequential(suite); - const result: SuiteResults = new SuiteResults(suite, testee); + const result: SuiteResult = new SuiteResult(suite); const first: TestScenario = order[0]; - await timeout('Initialize testbed', testee.connector.timeout, testee.initialize(first.program, first.args ?? []).catch((e: Error) => result.error = e)); + await timeout('Initialize testbed', testee.connector.timeout, testee.initialize(first.program, first.args ?? []).catch((e: Error) => result.error(e.message))); await this.runSuite(result, testee, order); this.reporter.report(result); @@ -114,10 +115,10 @@ export class Framework { await Promise.all(suites.map(async (suite: Suite) => { await Promise.all(suite.testees.map(async (testee: Testee) => { const order: TestScenario[] = suite.scheduler.sequential(suite); - const result: SuiteResults = new SuiteResults(suite, testee); + const result: SuiteResult = new SuiteResult(suite); const first: TestScenario = order[0]; - await timeout('Initialize testbed', testee.connector.timeout, testee.initialize(first.program, first.args ?? []).catch((e: Error) => result.error = e)); + await timeout('Initialize testbed', testee.connector.timeout, testee.initialize(first.program, first.args ?? []).catch((e: Error) => result.error(e.message))); await this.runSuite(result, testee, order); this.reporter.report(result); @@ -139,10 +140,10 @@ export class Framework { const order: TestScenario[][] = suite.scheduler.parallel(suite, suite.testees.length); await Promise.all(suite.testees.map(async (testee: Testee, i: number) => { // console.log(`scheduling on ${testee.name}`) - const result: SuiteResults = new SuiteResults(suite, testee); + const result: SuiteResult = new SuiteResult(suite); const first: TestScenario = order[i][0]; - await timeout('Initialize testbed', testee.connector.timeout, testee.initialize(first.program, first.args ?? []).catch((e: Error) => result.error = e)); + await timeout('Initialize testbed', testee.connector.timeout, testee.initialize(first.program, first.args ?? []).catch((e: Error) => result.error(e.message))); for (let j = i; j < order.length; j += suite.testees.length) { await this.runSuite(result, testee, order[j]); @@ -159,7 +160,7 @@ export class Framework { this.reporter.results(t1 - t0); } - private async runSuite(result: SuiteResults, testee: Testee, order: TestScenario[]) { + private async runSuite(result: SuiteResult, testee: Testee, order: TestScenario[]) { for (const test of order) { await testee.describe(test, result, this.runs); } diff --git a/src/framework/Testee.ts b/src/framework/Testee.ts index 0814a0c..2143e42 100644 --- a/src/framework/Testee.ts +++ b/src/framework/Testee.ts @@ -9,10 +9,11 @@ import {TestScenario} from './scenario/TestScenario'; import {OutofPlaceSpecification, PlatformType, TestbedSpecification} from '../testbeds/TestbedSpecification'; import {CompileOutput, CompilerFactory} from '../manage/Compiler'; import {WABT} from '../util/env'; -import {Completion, expect, ScenarioResult, SuiteResults} from '../reporter/Reporter'; +import {Outcome} from '../reporter/describers/Describer'; import {WASM} from '../sourcemap/Wasm'; import {DummyProxy} from '../testbeds/Emulator'; -import {Result} from '../reporter/Result'; +import {ScenarioResult, Skipped, StepOutcome, SuiteResult} from '../reporter/Results'; +import {Verifier} from './Verifier'; export function timeout(label: string, time: number, promise: Promise): Promise { if (time === 0) { @@ -46,14 +47,14 @@ export function getValue(object: any, field: string): any { } export enum Target { - supervisor, - proxy + supervisor = 'supervisor', + proxy = 'proxy' } export class Testee { // TODO unified with testbed interface /** The current state for each described test */ - private states: Map = new Map(); + private states: Map = new Map(); /** Factory to establish new connections to VMs */ public readonly connector: TestbedFactory; @@ -132,16 +133,16 @@ export class Testee { // TODO unified with testbed interface } private run(name: string, limit: number, fn: () => Promise) { - return timeout(name, limit, fn()); + return timeout(name, limit, fn()); } private step(name: string, limit: number, fn: () => Promise) { - return timeout(name, limit, fn()); + return timeout(name, limit, fn()); } - public async describe(description: TestScenario, suiteResult: SuiteResults, runs: number = 1) { + public async describe(description: TestScenario, suiteResult: SuiteResult, runs: number = 1) { const testee = this; - const scenarioResult: ScenarioResult = new ScenarioResult(description, testee); + const scenarioResult: ScenarioResult = new ScenarioResult(description); if (description.skip) { return; @@ -154,11 +155,11 @@ export class Testee { // TODO unified with testbed interface await this.run('Check for failing dependencies', testee.timeout, async function () { const failedDependencies: TestScenario[] = testee.failedDependencies(description); if (failedDependencies.length > 0) { - testee.states.set(description.title, new Result('Skipping', 'Test has failing dependencies', Completion.skipped)); + testee.states.set(description.title, new Skipped('Skipping', 'Test has failing dependencies')); throw new Error(`Skipped: failed dependent tests: ${failedDependencies.map(dependence => dependence.title)}`); } }).catch((e: Error) => { - scenarioResult.error = e; + scenarioResult.error(e.message); }); await this.run('Compile and upload program', testee.connector.timeout, async function () { @@ -169,16 +170,16 @@ export class Testee { // TODO unified with testbed interface const compiled: CompileOutput = await new CompilerFactory(WABT).pickCompiler(description.program).compile(description.program); try { - await timeout(`uploading module`, testee.timeout, testee.bed()!.sendRequest(new SourceMap.Mapping(), Message.updateModule(compiled.file))).catch((e) => Promise.reject(e)); + await timeout(`uploading module`, testee.timeout, testee.bed()!.sendRequest(new SourceMap.Mapping(), Message.updateModule(compiled.file))).catch((e) => Promise.reject(e)); testee.current = description.program; } catch (e) { await testee.initialize(description.program, description.args ?? []).catch((o) => Promise.reject(o)); } }).catch((e: Error | string) => { if (typeof e === 'string') { - scenarioResult.error = new Error(e); + scenarioResult.error(e); } else { - scenarioResult.error = e; + scenarioResult.error(e.toString()); } }); @@ -186,14 +187,14 @@ export class Testee { // TODO unified with testbed interface map = await testee.mapper.map(description.program); }).catch((e: Error | string) => { if (typeof e === 'string') { - scenarioResult.error = new Error(e); + scenarioResult.error(e); } else { - scenarioResult.error = e; + scenarioResult.error(e.toString()); } }); - if (scenarioResult.error) { - suiteResult.scenarios.push(scenarioResult); + if (scenarioResult.outcome === Outcome.error) { + suiteResult.add(scenarioResult); return; } @@ -205,62 +206,60 @@ export class Testee { // TODO unified with testbed interface await this.run('resetting before retry', testee.timeout, async function () { await testee.reset(testee.testbed); }).catch((e: Error) => { - scenarioResult.error = e; + scenarioResult.error(e.toString()); }); } for (const step of description.steps ?? []) { /** Perform the step and check if expectations were met */ await this.step(step.title, testee.timeout, async function () { - let result: Result = new Result(step.title, 'incomplete'); + const verifier: Verifier = new Verifier(step); if (testee.bed(step.target ?? Target.supervisor) === undefined) { - testee.states.set(description.title, result); - result.error('Cannot run test: no debugger connection.'); - testee.states.set(description.title, result); + testee.states.set(description.title, verifier.error('Cannot run test: no debugger connection.')); return; } - let actual: Object | void; + let actual: object | void; if (step.instruction.kind === Kind.Action) { - actual = await timeout(`performing action . ${step.title}`, testee.timeout, + actual = await timeout(`performing action . ${step.title}`, testee.timeout, step.instruction.value.act(testee)).catch((err) => { - result.error(err); + testee.states.set(description.title, verifier.error(err)); + return; }); } else { actual = await testee.recoverable(testee, step.instruction.value, map, - (testee, req, map) => timeout(`sending instruction ${req.type}`, testee.timeout, + (testee, req, map) => timeout(`sending instruction ${req.type}`, testee.timeout, testee.bed(step.target ?? Target.supervisor)!.sendRequest(map, req)), (testee) => testee.run(`Recover: re-initialize ${testee.testbed?.name}`, testee.connector.timeout, async function () { await testee.initialize(description.program, description.args ?? []).catch((o) => { return Promise.reject(o) }); }), 1).catch((e: Error) => { - result.completion = (e.message.includes('timeout')) ? Completion.timedout : Completion.error; - result.description = e.message; + const result = new StepOutcome(step); + testee.states.set(description.title, result.update((e.message.includes('timeout')) ? Outcome.timedout : Outcome.error, e.message)); }); } - if (result.completion === Completion.uncommenced) { - result = expect(step, actual, previous); - } + const result = verifier.verify(actual, previous); if (actual !== undefined) { previous = actual; } testee.states.set(description.title, result); - scenarioResult.results.push(result); + scenarioResult.add(result); }); } - suiteResult.scenarios.push(scenarioResult); + suiteResult.add(scenarioResult); } } + /* eslint @typescript-eslint/no-explicit-any: off */ private async recoverable(testee: Testee, step: Request, map: SourceMap.Mapping, - attempt: (t: Testee, req: Request, m: SourceMap.Mapping) => Promise, + attempt: (t: Testee, req: Request, m: SourceMap.Mapping) => Promise, recover: (t: Testee) => Promise, - retries: number = 0): Promise { - let result: Object | void = undefined; + retries: number = 0): Promise { + let result: object | void = undefined; let error; while (0 <= retries && result === undefined) { result = await attempt(testee, step, map).catch(async (err) => { @@ -281,7 +280,7 @@ export class Testee { // TODO unified with testbed interface if (instance === undefined) { this.framework.reporter.error('Cannot run test: no debugger connection.'); // todo } else { - await timeout('resetting vm', this.timeout, this.testbed!.sendRequest(new SourceMap.Mapping(), Message.reset)); + await timeout('resetting vm', this.timeout, this.testbed!.sendRequest(new SourceMap.Mapping(), Message.reset)); } } @@ -293,8 +292,8 @@ export class Testee { // TODO unified with testbed interface private failedDependencies(description: TestScenario): TestScenario[] { return (description?.dependencies ?? []).filter(dependence => { if (this.states.get(dependence.title)) { - const c = this.states.get(dependence.title)!.completion; - return !(c === Completion.succeeded || c === Completion.uncommenced); + const c = this.states.get(dependence.title)!.outcome; + return !(c === Outcome.succeeded || c === Outcome.uncommenced); } else { return false; } diff --git a/src/framework/Verifier.ts b/src/framework/Verifier.ts new file mode 100644 index 0000000..8a155bd --- /dev/null +++ b/src/framework/Verifier.ts @@ -0,0 +1,128 @@ +import {Behaviour, Description, Step} from './scenario/Step'; +import {StepOutcome} from '../reporter/Results'; +import {getValue} from './Testee'; +import {Outcome} from '../reporter/describers/Describer'; +import {bold} from 'ansi-colors'; + +// decorator for Step class +export class Verifier { + public readonly step: Step; + + constructor(step: Step) { + this.step = step; + } + + public verify(actual: Object | void, previous?: Object): StepOutcome { + let result = new StepOutcome(this.step).update(Outcome.succeeded); + for (const expectation of this.step.expected ?? []) { + for (const [field, entry] of Object.entries(expectation)) { + try { + const value = getValue(actual, field); + + if (entry.kind === 'primitive') { + result = this.expectPrimitive(value, entry.value); + } else if (entry.kind === 'description') { + result = this.expectDescription(value, entry.value); + } else if (entry.kind === 'comparison') { + result = this.expectComparison(actual, value, entry.value, entry.message); + } else if (entry.kind === 'behaviour') { + if (previous === undefined) { + return this.error('Invalid test: no [previous] to compare behaviour to.'); + } + result = this.expectBehaviour(value, getValue(previous, field), entry.value); + } + } catch { + return this.error(`Failure: ${JSON.stringify(actual)} state does not contain '${field}'.`); + } + + if (result.outcome !== Outcome.succeeded) { + return result; + } + } + } + return result; + } + + public error(clarification: string): StepOutcome { + const result: StepOutcome = new StepOutcome(this.step); + result.update(Outcome.succeeded); + return result.update(Outcome.error, clarification); + } + + private expectPrimitive(actual: T, expected: T): StepOutcome { + const result: StepOutcome = new StepOutcome(this.step); + if (deepEqual(actual, expected)) { + result.update(Outcome.succeeded); + } else { + result.update(Outcome.failed, `Expected ${bold(`${expected}`)} got ${bold(`${actual}`)}`); + } + return result; + } + + private expectDescription(actual: T, value: Description): StepOutcome { + const result: StepOutcome = new StepOutcome(this.step); + if ((value === Description.defined && actual !== undefined) || + value === Description.notDefined && actual === undefined) { + result.update(Outcome.succeeded); + } else { + result.update(Outcome.failed, value === Description.defined ? 'Should exist' : 'Unexpected field'); + } + return result; + } + + private expectComparison(state: Object | void, actual: T, comparator: (state: Object, value: T) => boolean, message?: string): StepOutcome { + const result: StepOutcome = new StepOutcome(this.step); + if (state === undefined) { + result.update(Outcome.failed, `Got unexpected ${state}`); + return result; + } + + if (comparator(state, actual)) { + result.update(Outcome.succeeded); + } else { + result.update(Outcome.failed, `Fail: ${message}`); + } + + return result; + } + + private expectBehaviour(actual: any, previous: any, behaviour: Behaviour): StepOutcome { + const result: StepOutcome = new StepOutcome(this.step); + switch (behaviour) { + case Behaviour.unchanged: + if (deepEqual(actual, previous)) { + result.update(Outcome.succeeded); + } else { + result.update(Outcome.failed, `Expected ${actual} to equal ${previous}`); + } + break; + case Behaviour.changed: + if (!deepEqual(actual, previous)) { + result.update(Outcome.succeeded); + } else { + result.update(Outcome.failed, `Expected ${actual} to be different from ${previous}`); + } + break; + case Behaviour.increased: + if (actual > previous) { + result.update(Outcome.succeeded); + } else { + result.update(Outcome.failed, `Expected ${actual} to be greater than ${previous}`); + } + break; + case Behaviour.decreased: + if (actual < previous) { + result.update(Outcome.succeeded); + } else { + result.update(Outcome.failed, `Expected ${actual} to be less than ${previous}`); + } + break; + } + return result; + } +} + +/* eslint @typescript-eslint/no-explicit-any: off */ +function deepEqual(a: any, b: any): boolean { + return a === b || (isNaN(a) && isNaN(b)); +} diff --git a/src/manage/Compiler.ts b/src/manage/Compiler.ts index f9dfa29..9281a13 100644 --- a/src/manage/Compiler.ts +++ b/src/manage/Compiler.ts @@ -406,4 +406,3 @@ function getIndex(line: string): number { function getName(line: string): string { return find(/-> "([^"]+)"/, line); } - diff --git a/src/reporter/Reporter.ts b/src/reporter/Reporter.ts index 7eb3287..f1cdbd7 100644 --- a/src/reporter/Reporter.ts +++ b/src/reporter/Reporter.ts @@ -1,145 +1,51 @@ -import {Suite} from '../framework/Framework'; -import {Step} from '../framework/scenario/Step'; -import {getValue, Testee} from '../framework/Testee'; -import {blue, bold, green, inverse, red, yellow} from 'ansi-colors'; +import {SuiteResult} from './Results'; import {Archiver} from '../framework/Archiver'; -import {TestScenario} from '../framework/scenario/TestScenario'; +import {Style} from './Style'; +import {Verbosity} from './index'; import {version} from '../../package.json'; +import {green, red, yellow} from 'ansi-colors'; +import {Outcome, SilentDescriber} from './describers/Describer'; import {indent} from '../util/printing'; -import {Result} from './Result'; -import {Verbosity} from './index'; -import {Style} from './Style'; - -export enum Completion { - uncommenced = 'not started', // test hasn't started - succeeded = 'success', // test succeeded - failed = 'Failure: ', // test failed - timedout = 'timed out', // test failed - error = 'error: ', // test was unable to complete - skipped = 'skipped' // test has failing dependencies -} - -export function expect(step: Step, actual: Object | void, previous?: Object): Result { - const result: Result = new Result(step.title, ''); - result.completion = Completion.succeeded; - for (const expectation of step.expected ?? []) { - for (const [field, entry] of Object.entries(expectation)) { - try { - const value = getValue(actual, field); - - if (entry.kind === 'primitive') { - result.expectPrimitive(value, entry.value); - } else if (entry.kind === 'description') { - result.expectDescription(value, entry.value); - } else if (entry.kind === 'comparison') { - result.expectComparison(actual, value, entry.value, entry.message); - } else if (entry.kind === 'behaviour') { - if (previous === undefined) { - result.error('Invalid test: no [previous] to compare behaviour to.'); - return result; - } - result.expectBehaviour(value, getValue(previous, field), entry.value); - } - } catch (e) { - result.error(`Failure: ${JSON.stringify(actual)} state does not contain '${field}'.`); - return result; - } - } - } - return result; -} - -export class SuiteResults { - public suite: Suite; - public testee: Testee; - public scenarios: ScenarioResult[] = []; - public error?: Error | string; - - constructor(suite: Suite, testee: Testee) { - this.suite = suite; - this.testee = testee; - } - - title(): string { - return this.suite.title; - } - - passing(): boolean { - return this.scenarios.every((scenario) => scenario.passing()); - } - - failing(): boolean { - return this.error !== undefined || this.scenarios.some((scenario) => scenario.failing()); - } - - skipped(): boolean { - return this.scenarios.every((scenario) => scenario.skipped()); - } -} - -export class ScenarioResult { - public test: TestScenario; - public testee: Testee; - public results: Result[] = []; - public error?: Error; - - - constructor(test: TestScenario, testee: Testee) { - this.test = test; - this.testee = testee; - } - - title(): string { - return `${this.test.title}`; - } - - steps(): string[] { - return this.results.map((result) => result.toString()); - } - - passing(): boolean { - return this.error === undefined && this.results.every((result) => result.completion === Completion.succeeded); - } - - failing(): boolean { - return this.error !== undefined || this.results.some((result) => result.completion === Completion.failed); - } - - skipped(): boolean { - return this.results.every((result) => result.completion === Completion.uncommenced); - } - - report(index: number, level: number): void { - // console.log(indent(level) + inverse(blue(` ${index + 1} `)) + ' ' + bold(this.title()) + '\n'); - console.log(indent(level) + bold(blue(`scenario.`)) + ' ' + bold(this.title()) + ' ' + bold(blue(`(${index + 1})`)) + '\n'); - if (this.error) { - console.log(red(`${indent(level)}${bold(inverse(red(' ERROR ')))} ${this.error.message.trim().replace(/\n/g, `\n${indent(level)}`)}`)); - } else { - this.results.forEach((result) => { - result.report(level); - }); - } - console.log() +import { + MinimalSuiteDescriber, + NormalSuiteDescriber, + ShortSuiteDescriber, + SuiteDescriber +} from './describers/SuiteDescribers'; + +function describer(verbosity: Verbosity, item: SuiteResult): SuiteDescriber { + switch (verbosity) { + case Verbosity.none: + return new SilentDescriber(item); + case Verbosity.minimal: + return new MinimalSuiteDescriber(item); + case Verbosity.short: + return new ShortSuiteDescriber(item); + case Verbosity.normal: + case Verbosity.more: + case Verbosity.all: + case Verbosity.debug: + default: + return new NormalSuiteDescriber(item); } } -// const r Result = expect(e: Expected) <-- replaces the function in Testee todo - export class Reporter { private output: string = ''; private indentationLevel: number = 2; - private style: Style; - - private suites: SuiteResults[] = []; + private suites: SuiteResult[] = []; private archiver: Archiver; - private verbosity: Verbosity = Verbosity.short; + private readonly style: Style; + + private readonly verbosity: Verbosity; - constructor(style: Style) { + constructor(style: Style, verbosity: Verbosity = Verbosity.normal) { this.style = style; + this.verbosity = verbosity; this.archiver = new Archiver(`${process.env.TESTFILE?.replace('.asserts.wast', '.wast') ?? 'suite'}.${Date.now()}.log`); this.archiver.set('date', new Date(Date.now()).toISOString()); } @@ -149,33 +55,19 @@ export class Reporter { } general() { - console.log(this.indent() + this.style.colors.highlight(this.style.bullet) + this.style.colors.highlight('latch') + this.style.emph(' General information')); + console.log(this.indent() + this.style.colors.highlight(this.style.bullet) + this.style.colors.highlight('latch.') + this.style.emph(' General information')); // console.log(blue(`${this.indent()}===================`)); console.log(this.indent() + ' '.repeat(2) + this.style.emph('version') + ' '.repeat(5) + version); console.log(this.indent() + ' '.repeat(2) + this.style.emph('archive') + ' '.repeat(5) + this.archiver.archive); console.log(); } - report(suiteResult: SuiteResults) { + report(suiteResult: SuiteResult) { this.suites.push(suiteResult); - const status = (suiteResult.error ? this.style.colors.error(this.style.labels.error) : - (suiteResult.passing() ? this.style.colors.success(this.style.labels.suiteSuccess) : this.style.colors.failure(this.style.labels.failure))); - console.log(this.indent() + this.style.colors.highlight(this.style.bullet) + this.style.colors.highlight('suite') + ` ${this.style.emph(suiteResult.title())}${(this.verbosity === Verbosity.minimal) ? ' ' + status : ''}`); - if (this.verbosity > Verbosity.minimal) { - console.log(this.indent() + ' '.repeat(2) + this.style.emph('testbed') + ' '.repeat(5) + suiteResult.testee.name); - console.log(this.indent() + ' '.repeat(2) + this.style.emph('scenarios') + ' '.repeat(3) + suiteResult.scenarios.length); - console.log(this.indent() + ' '.repeat(2) + this.style.emph('actions') + ' '.repeat(5) + suiteResult.suite.scenarios.flatMap((scenario) => scenario.steps ?? []).flat().length); //.reduce((total, count) => total + count)); - console.log(this.indent() + ' '.repeat(2) + this.style.emph('status') + ' '.repeat(6) + status); - } - console.log(); - if (this.verbosity >= Verbosity.normal) { - suiteResult.scenarios.forEach((scenario, index) => { - scenario.report(index, this.indentationLevel + 1); - }); - } else if (this.verbosity > Verbosity.minimal) { - if (suiteResult.error) { - console.log(this.indent() + ' '.repeat(2) + red(suiteResult.error.toString())); - } + const report: string[] = describer(this.verbosity, suiteResult).describe(this.style); + + for (const line of report) { + console.log(this.indent() + line); } console.log(); } @@ -183,37 +75,34 @@ export class Reporter { results(time: number) { this.archiver.set('duration (ms)', Math.round(time)); - const passing = this.suites.flatMap((suite) => suite.scenarios).filter((scenario) => scenario.passing()).length; - const failing = this.suites.flatMap((suite) => suite.scenarios).filter((scenario) => scenario.failing()).length; - const skipped = this.suites.flatMap((suite) => suite.scenarios).filter((scenario) => scenario.skipped()).length; + const passing = this.suites.flatMap((suite) => suite.outcomes()).filter((scenario) => scenario.outcome === Outcome.succeeded).length; + const failing = this.suites.flatMap((suite) => suite.outcomes()).filter((scenario) => scenario.outcome === Outcome.failed).length; + const skipped = this.suites.flatMap((suite) => suite.outcomes()).filter((scenario) => scenario.outcome === Outcome.skipped).length; - const scs = this.suites.flatMap((suite) => suite.scenarios); + const scs = this.suites.flatMap((suite) => suite.outcomes()); - this.suites.flatMap((suite) => suite.scenarios).filter((scenario) => scenario.failing()).forEach((scenario) => this.archiver.extend('failures', scenario.title())); - this.suites.flatMap((suite) => suite.scenarios).filter((scenario) => scenario.passing()).forEach((scenario) => this.archiver.extend('passes', scenario.title())); + this.suites.flatMap((suite) => suite.outcomes()).filter((scenario) => scenario.outcome === Outcome.failed).forEach((scenario) => this.archiver.extend('failures', scenario.name)); + this.suites.flatMap((suite) => suite.outcomes()).filter((scenario) => scenario.outcome === Outcome.succeeded).forEach((scenario) => this.archiver.extend('passes', scenario.name)); this.archiver.set('passed scenarios', passing); this.archiver.set('skipped scenarios', skipped); this.archiver.set('failed scenarios', failing); - console.log(this.indent() + this.style.colors.highlight(this.style.bullet) + this.style.colors.highlight('results') + this.style.emph(' Overview')); + console.log(this.indent() + this.style.colors.highlight(this.style.bullet) + this.style.colors.highlight('results.') + this.style.emph(' Overview')); console.log(); this.indentationLevel += 1; - const sc = this.suites.filter((suite) => suite.passing()).length; + const sc = this.suites.filter((suite) => suite.outcome === Outcome.succeeded).length; const tl = this.suites.length; - const psa = this.suites.flatMap((suite) => suite.scenarios).flatMap((scenario) => - scenario.results.filter((result) => - result.completion === Completion.succeeded).length).reduce((acc, val) => acc + val, 0); - const fa = this.suites.flatMap((suite) => suite.scenarios).flatMap((scenario) => - scenario.results.filter((result) => - result.completion === Completion.failed).length).reduce((acc, val) => acc + val, 0); - const timeouts = this.suites.flatMap((suite) => suite.scenarios).flatMap((scenario) => - scenario.results.filter((result) => - result.completion === Completion.timedout).length).reduce((acc, val) => acc + val, 0); - const total = this.suites.flatMap((suite) => suite.scenarios).flatMap((scenario) => - scenario.test.steps?.length ?? 0).reduce((acc, val) => acc + val, 0); + const psa = this.suites.flatMap((suite) => suite.outcomes()).flatMap((scenario) => + scenario.outcomes().filter((result) => + result.outcome === Outcome.succeeded).length).reduce((acc, val) => acc + val, 0); + const timeouts = this.suites.flatMap((suite) => suite.outcomes()).flatMap((scenario) => + scenario.outcomes().filter((result) => + result.outcome === Outcome.timedout).length).reduce((acc, val) => acc + val, 0); + const total = this.suites.flatMap((suite) => suite.outcomes()).flatMap((scenario) => + scenario.outcomes().length ?? 0).reduce((acc, val) => acc + val, 0); const len: number = 12; const pss = [`${sc} passing`, `${passing} passing`, `${psa} passing`] @@ -245,8 +134,4 @@ export class Reporter { test(title: string) { this.output += ` test: ${title}\n`; } - - step(result: Result) { - this.output += ` ${result.toString()}\n`; - } -} +} \ No newline at end of file diff --git a/src/reporter/Result.ts b/src/reporter/Result.ts deleted file mode 100644 index 724b0be..0000000 --- a/src/reporter/Result.ts +++ /dev/null @@ -1,115 +0,0 @@ -import {bold, green, inverse, red, reset, yellow} from 'ansi-colors'; -import {indent} from '../util/printing'; -import {Behaviour, Description} from '../framework/scenario/Step'; -import {Completion} from './Reporter'; - -export class Result { - public completion: Completion; // completion status of the step - public name: string; // name of the step - public description: string; - - constructor(name: string, description: string, completion?: Completion) { - this.name = name; - this.description = description; - this.completion = completion ?? Completion.uncommenced; - } - - report(level: number) { - console.log(reset(`${indent(level)}${this}`)); - } - - toString(): string { - switch (this.completion) { - case Completion.succeeded: - return `${bold(inverse(green(' PASS ')))} ${this.name}`; - case Completion.uncommenced: - return `${bold(inverse(yellow(' SKIP ')))} ${this.name}`; - case Completion.error: - case Completion.failed: - default: - return `${bold(inverse(red(' FAIL ')))} ${this.name}\n ${red(this.completion)}${red(this.description)}`; - - } - } - - error(description: string) { - this.completion = Completion.error; - this.description = description; - } - - public expectPrimitive(actual: T, expected: T): void { - // this.completion = deepEqual(actual, expected) ? Completion.succeeded : Completion.failed; - if (deepEqual(actual, expected)) { - this.completion = Completion.succeeded; - } else { - this.completion = Completion.failed; - this.description = `Expected ${bold(`${expected}`)} got ${bold(`${actual}`)}`; - } - } - - public expectDescription(actual: T, value: Description): void { - if ((value === Description.defined && actual !== undefined) || - value === Description.notDefined && actual === undefined) { - this.completion = Completion.succeeded; - } else { - this.completion = Completion.failed; - this.description = value === Description.defined ? 'Should exist' : 'Unexpected field'; - } - } - - public expectComparison(state: Object | void, actual: T, comparator: (state: Object, value: T) => boolean, message?: string): void { - if (state === undefined) { - this.completion = Completion.failed; - this.description = `Got unexpected ${state}`; - return; - } - - if (comparator(state, actual)) { - this.completion = Completion.succeeded; - } else { - this.completion = Completion.failed; - this.description = 'custom comparator failed'; - } - } - - public expectBehaviour(actual: any, previous: any, behaviour: Behaviour): void { - switch (behaviour) { - case Behaviour.unchanged: - if (deepEqual(actual, previous)) { - this.completion = Completion.succeeded; - } else { - this.completion = Completion.failed; - this.description = `Expected ${actual} to equal ${previous}` - } - break; - case Behaviour.changed: - if (!deepEqual(actual, previous)) { - this.completion = Completion.succeeded; - } else { - this.completion = Completion.failed; - this.description = `Expected ${actual} to be different from ${previous}` - } - break; - case Behaviour.increased: - if (actual > previous) { - this.completion = Completion.succeeded; - } else { - this.completion = Completion.failed; - this.description = `Expected ${actual} to be greater than ${previous}` - } - break; - case Behaviour.decreased: - if (actual < previous) { - this.completion = Completion.succeeded; - } else { - this.completion = Completion.failed; - this.description = `Expected ${actual} to be less than ${previous}` - } - break; - } - } -} - -function deepEqual(a: any, b: any): boolean { - return a === b || (isNaN(a) && isNaN(b)); -} diff --git a/src/reporter/Results.ts b/src/reporter/Results.ts new file mode 100644 index 0000000..0704a5f --- /dev/null +++ b/src/reporter/Results.ts @@ -0,0 +1,189 @@ +import {Step} from '../framework/scenario/Step'; +import {Outcome} from './describers/Describer'; +import {TestScenario} from '../framework/scenario/TestScenario'; +import {Suite} from '../framework/Framework'; + +export interface Result { + outcome: Outcome; + clarification: string; + + readonly name: string; + readonly checks: number; + + readonly testbed?: string; + + update(outcome?: Outcome, clarification?: string): Result; + + error(clarification: string): Result; +} + +interface AggregateResult extends Result { + add(outcomes: Result): void; + aggregate(outcomes: Result[]): void; + outcomes(): Result[]; +} + +abstract class AbstractAggregateResult implements AggregateResult { + abstract outcome: Outcome; + abstract clarification: string; + abstract readonly name: string; + abstract readonly testbed?: string; + abstract readonly checks: number; + + protected abstract subOutcomes: Result[]; + + abstract update(outcome?: Outcome, clarification?: string): AggregateResult; + + error(clarification: string): AggregateResult { + this.outcome = Outcome.error; + this.clarification = clarification; + return this; + } + + add(outcome: Result) { + this.subOutcomes.push(outcome); + this.outcome = this.check(); + } + + aggregate(outcomes: Result[]) { + this.subOutcomes = outcomes; + this.outcome = this.check(); + } + + outcomes(): Result[] { + return this.subOutcomes; + } + + private check(): Outcome { + if (this.outcome === Outcome.error) { + return this.outcome; + } + + if (this.passing()) { + return Outcome.succeeded; + } else if (this.failing()) { + return Outcome.failed; + } + + return Outcome.skipped; + } + + private passing(): boolean { + return this.subOutcomes.every((outcome) => outcome.outcome === Outcome.succeeded); + } + + private failing(): boolean { + return this.subOutcomes.some((outcome) => outcome.outcome === Outcome.failed); + } +} + +export class ScenarioResult extends AbstractAggregateResult { + outcome: Outcome; + clarification: string; + readonly name: string; + readonly testbed?: string; + readonly checks: number; + + protected subOutcomes: StepOutcome[]; + + constructor(scenario: TestScenario) { + super(); + this.name = scenario.title; + this.checks = scenario.steps?.map(step => step.expected?.length ?? 0).reduce((a, b) => a + b) ?? 0; + this.testbed = scenario.steps?.[0].target; + + this.outcome = Outcome.uncommenced; + this.clarification = ''; + + this.subOutcomes = []; + } + + update(outcome?: Outcome, clarification?: string): ScenarioResult { + this.outcome = outcome ?? this.outcome; + this.clarification = clarification ?? this.clarification; + return this; + } +} + +export class SuiteResult extends AbstractAggregateResult { + outcome: Outcome; + clarification: string; + readonly name: string; + readonly testbed?: string; + readonly checks: number; + + protected subOutcomes: ScenarioResult[]; + + constructor(suite: Suite) { + super(); + this.name = suite.title; + this.checks = suite.scenarios.map(scenario => scenario.steps?.map(step => step.expected?.length ?? 0).reduce((a, b) => a + b) ?? 0).reduce((a, b) => a + b) ?? 0; + this.testbed = suite.scenarios?.[0].steps?.[0].target; + + this.outcome = Outcome.uncommenced; + this.clarification = ''; + + this.subOutcomes = []; + } + + update(outcome?: Outcome, clarification?: string): SuiteResult { + this.outcome = outcome ?? this.outcome; + this.clarification = clarification ?? this.clarification; + return this; + } + + outcomes(): ScenarioResult[] { + return this.subOutcomes; + } +} + +export class StepOutcome implements Result { + outcome: Outcome; + clarification: string; + readonly name: string; + readonly testbed?: string; + readonly checks: number; + + constructor(step: Step) { + this.outcome = Outcome.uncommenced; + this.name = step.title; + this.clarification = ''; + this.testbed = step.target; + this.checks = step.expected?.length ?? 0; + } + + public update(outcome?: Outcome, clarification?: string): StepOutcome { + this.outcome = outcome ?? this.outcome; + this.clarification = clarification ?? this.clarification; + return this; + } + + public error(clarification: string): StepOutcome { + this.outcome = Outcome.error; + this.clarification = clarification; + return this; + } +} + +export class Skipped implements Result { + outcome: Outcome; + clarification: string; + readonly name: string; + readonly testbed?: string; + readonly checks: number; + + constructor(title: string, clarification: string) { + this.outcome = Outcome.skipped; + this.name = title; + this.clarification = clarification; + this.checks = 0; + } + + public update(): Skipped { + return this; + } + + public error(): Skipped { + return this; + } +} diff --git a/src/reporter/Style.ts b/src/reporter/Style.ts index a920695..be821df 100644 --- a/src/reporter/Style.ts +++ b/src/reporter/Style.ts @@ -10,11 +10,13 @@ interface Colors { success: Styler; skipped: Styler; failure: Styler; + failureMessage: Styler; error: Styler; } interface Labels { suiteSuccess: string; + suiteSkipped: string; success: string; skipped: string; @@ -51,14 +53,16 @@ export class Plain implements Style { end = ''; emph = (s: string) => bold(s); colors: Colors = { - highlight: (s: string) => blue(s), + highlight: (s: string) => bold(blue(s)), success: (s: string) => inverse(bold(green(s))), skipped: (s: string) => inverse(bold(yellow(s))), failure: (s: string) => inverse(bold(red(s))), + failureMessage: (s: string) => red(s), error: (s: string) => inverse(bold(red(s))) }; labels: Labels = { suiteSuccess: ' PASSED ', + suiteSkipped: ' SKIPPED ', success: ' PASS ', skipped: ' SKIP ', failure: ' FAIL ', diff --git a/src/reporter/describers/Describer.ts b/src/reporter/describers/Describer.ts new file mode 100644 index 0000000..b476e71 --- /dev/null +++ b/src/reporter/describers/Describer.ts @@ -0,0 +1,56 @@ +import {Result, StepOutcome} from '../Results'; +import {Style} from '../Style'; + +export interface Describable { + readonly item: R; + + describe(style: Style): string[]; +} + +export abstract class Describer implements Describable { + public readonly item: R; + + constructor(item: R) { + this.item = item; + } + + abstract describe(style: Style): string[]; +} + +export class SilentDescriber extends Describer { + describe(): string[] { + return []; + } +} + +export class StepDescriber extends Describer { + constructor(outcome: Result) { + super(outcome); + } + + describe(style: Style): string[] { + switch (this.item.outcome) { + case Outcome.succeeded: + return [`${style.colors.success(style.labels.success)} ${this.item.name}`]; + case Outcome.uncommenced: + case Outcome.skipped: + return [`${style.colors.skipped(style.labels.skipped)} ${this.item.name}`]; + case Outcome.error: + case Outcome.failed: + default: + return [`${style.colors.failure(style.labels.failure)} ${this.item.name}\n ${style.colors.failureMessage(this.item.outcome + this.item.clarification)}`]; + } + } +} + +export enum Outcome { + uncommenced = 'not started', // test hasn't started + succeeded = 'success', // test succeeded + failed = 'failure: ', // test failed + timedout = 'timed out', // test failed + error = 'error: ', // test was unable to complete + skipped = 'skipped' // test has failing dependencies +} + +// const r Result = expect(e: Expected) <-- replaces the function in Testee todo + diff --git a/src/reporter/describers/ScenarioDescribers.ts b/src/reporter/describers/ScenarioDescribers.ts new file mode 100644 index 0000000..22d1157 --- /dev/null +++ b/src/reporter/describers/ScenarioDescribers.ts @@ -0,0 +1,44 @@ +// decorator class for minimal describers +import {Outcome, Describer, StepDescriber} from './Describer'; +import {ScenarioResult} from '../Results'; +import {Style} from '../Style'; + +abstract class ScenarioDescriber extends Describer { + protected readonly label?: string; + + constructor(scenario: ScenarioResult, label?: string) { + super(scenario); + this.label = label; + } +} + +export class MinimalScenarioDescriber extends ScenarioDescriber { + describe(style: Style): string[] { + const report: string[] = []; + report.push(style.colors.highlight(`scenario.`) + ' ' + style.emph(this.item.name) + ' ' + style.colors.highlight(this.label ?? '') + '\n'); + return report; + } +} + +export class ShortScenarioDescriber extends MinimalScenarioDescriber { + describe(style: Style): string[] { + let report: string[] = super.describe(style); + if (this.item.outcome === Outcome.error) { + report.push(style.colors.failureMessage(`${style.colors.failure(style.labels.error)}`)); + report = report.concat(this.item.clarification.trim().split('\n')); + } + return report; + } +} + +export class NormalScenarioDescriber extends ShortScenarioDescriber { + describe(style: Style): string[] { + let report: string[] = super.describe(style); + if (this.item.outcome !== Outcome.error) { + this.item.outcomes().forEach((outcome) => { + report = report.concat(new StepDescriber(outcome).describe(style)); + }); + } + return report; + } +} diff --git a/src/reporter/describers/SuiteDescribers.ts b/src/reporter/describers/SuiteDescribers.ts new file mode 100644 index 0000000..73c7e4e --- /dev/null +++ b/src/reporter/describers/SuiteDescribers.ts @@ -0,0 +1,71 @@ +import {Outcome, Describer} from './Describer'; +import {SuiteResult} from '../Results'; +import {Style} from '../Style'; +import {red} from 'ansi-colors'; +import {NormalScenarioDescriber} from './ScenarioDescribers'; + +const table: (style: Style) => Map = + (style: Style) => new Map([ + [Outcome.error, style.colors.error(style.labels.error)], + [Outcome.succeeded, style.colors.success(style.labels.suiteSuccess)], + [Outcome.failed, style.colors.failure(style.labels.failure)], + [Outcome.skipped, style.colors.skipped(style.labels.suiteSkipped)]]); + +export abstract class SuiteDescriber extends Describer {} + +export class MinimalSuiteDescriber extends SuiteDescriber { + describe(style: Style): string[] { + const report: string[] = []; + const status = (this.item.outcome === Outcome.error ? style.colors.error(style.labels.error) : + (this.item.outcome === Outcome.succeeded ? style.colors.success(style.labels.suiteSuccess) : style.colors.failure(style.labels.failure))); + report.push(style.colors.highlight(style.bullet) + style.colors.highlight('suite.') + ` ${style.emph(this.item.name)} ${status}`); + return report; + } +} + +export class ShortSuiteDescriber extends SuiteDescriber { + describe(style: Style): string[] { + let report: string[] = []; + report = report.concat(this.overview(style)); + + if (this.item.outcome === Outcome.error) { + report.push(''); + report.push(' '.repeat(2) + red(this.item.clarification.toString())); + } + return report; + } + + protected overview(style: Style): string[] { + const overview: string[] = []; + const status = table(style).get(this.item.outcome) ?? style.colors.skipped(style.labels.suiteSkipped); + overview.push(style.colors.highlight(style.bullet) + style.colors.highlight('suite.') + ` ${style.emph(this.item.name)}`); + if (this.item.testbed) { + overview.push(' '.repeat(2) + style.emph('testbed') + ' '.repeat(5) + this.item.testbed); + } + overview.push(' '.repeat(2) + style.emph('scenarios') + ' '.repeat(3) + this.item.outcomes().length); + overview.push(' '.repeat(2) + style.emph('actions') + ' '.repeat(5) + this.item.outcomes().flatMap((scenario) => scenario.outcomes() ?? []).flat().length); //.reduce((total, count) => total + count)); + overview.push(' '.repeat(2) + style.emph('status') + ' '.repeat(6) + status); + return overview; + } +} + +export class NormalSuiteDescriber extends ShortSuiteDescriber { + describe(style: Style): string[] { + + let report: string[] = []; + report = report.concat(this.overview(style)); + report.push(''); + + this.item.outcomes().forEach((scenario, index) => { + report = report.concat(new NormalScenarioDescriber(scenario, `(#${index + 1})`).describe(style)); + report.push(''); + }); + + if (this.item.outcome === Outcome.error) { + report.push(' '.repeat(2) + red(this.item.clarification.toString())); + } + + return report; + } + +} diff --git a/src/util/retry.ts b/src/util/retry.ts index a2f7e78..4670bc3 100644 --- a/src/util/retry.ts +++ b/src/util/retry.ts @@ -11,7 +11,6 @@ export function retry(promise: () => Promise, retries: number): Promise trying = ++attempt < retries; } } - reject(`exhausted number of retries (${retries})`); }); } \ No newline at end of file diff --git a/tests/examples/example.ts b/tests/examples/example.ts index f8f3ee1..71c690d 100644 --- a/tests/examples/example.ts +++ b/tests/examples/example.ts @@ -1,4 +1,5 @@ import { + Description, EmulatorSpecification, Expected, Framework, @@ -9,8 +10,7 @@ import { Step, Target, WASM -} from '../src/index'; -import {WARDuino} from '../src/debug/WARDuino'; +} from '../../src/index'; import dump = Message.dump; import stepOver = Message.stepOver; import step = Message.step; @@ -34,7 +34,7 @@ steps.push(new Invoker('func-unwind-by-br', [], undefined)); spec.test({ title: `Test with address_0.wast`, - program: 'tests/address.wast', + program: 'tests/examples/address.wast', dependencies: [], steps: steps }); @@ -46,7 +46,7 @@ debug.testee('emulator[:8522]', new EmulatorSpecification(8522)); debug.test({ title: 'Test STEP OVER', - program: 'tests/call.wast', + program: 'tests/examples/call.wast', steps: [{ title: 'Send DUMP command', instruction: {kind: Kind.Request, value: dump} @@ -74,13 +74,35 @@ debug.test({ }] }); +const DUMP: Step = { + title: 'Send DUMP command', + instruction: {kind: Kind.Request, value: Message.dump}, + expected: [ + {'pc': {kind: 'description', value: Description.defined} as Expected}, + { + 'breakpoints': { + kind: 'comparison', value: (state: Object, value: Array) => { + return value.length === 0; + }, message: 'list of breakpoints should be empty' + } as Expected> + }, + {'callstack[0].sp': {kind: 'primitive', value: -1} as Expected}, + {'callstack[0].fp': {kind: 'primitive', value: -1} as Expected}] +}; + +debug.test({ + title: 'Test DUMP blink', + program: `tests/examples/blink.wast`, + steps: [DUMP] +}); + const primitives = framework.suite('Test primitives'); primitives.testee('debug[:8700]', new EmulatorSpecification(8700)); primitives.test({ title: `Test store primitive`, - program: 'tests/dummy.wast', + program: 'tests/examples/dummy.wast', dependencies: [], steps: [{ title: 'CHECK: execution at start of main', @@ -115,7 +137,7 @@ oop.testee('supervisor[:8100] - proxy[:8150]', new OutofPlaceSpecification(8100, oop.test({ title: `Test store primitive`, - program: 'tests/dummy.wast', + program: 'tests/examples/dummy.wast', dependencies: [], steps: [ { @@ -161,4 +183,4 @@ oop.test({ }); -framework.run([spec, debug, primitives, oop]); +framework.run([spec, debug, primitives, oop]).then(() => process.exit(0));