fix(vitest): correctly report failed test files as failures in json reporter, export json reporter types (#5081)

Author: sheremet-va

packages/vitest/src/node/reporters/index.ts

@@ -23,6 +23,8 @@ export {
 }
 export type { BaseReporter, Reporter }
 
+export type { JsonAssertionResult, JsonTestResult, JsonTestResults } from './json'
+
 export const ReportersMap = {
   'default': DefaultReporter,
   'basic': BasicReporter,

packages/vitest/src/node/reporters/json.ts

@@ -22,7 +22,7 @@ const StatusMap: Record<TaskState, Status> = {
   todo: 'todo',
 }
 
-interface FormattedAssertionResult {
+export interface JsonAssertionResult {
   ancestorTitles: Array<string>
   fullName: string
   status: Status
@@ -32,18 +32,18 @@ interface FormattedAssertionResult {
   location?: Callsite | null
 }
 
-interface FormattedTestResult {
+export interface JsonTestResult {
   message: string
   name: string
   status: 'failed' | 'passed'
   startTime: number
   endTime: number
-  assertionResults: Array<FormattedAssertionResult>
+  assertionResults: Array<JsonAssertionResult>
   // summary: string
   // coverage: unknown
 }
 
-interface FormattedTestResults {
+export interface JsonTestResults {
   numFailedTests: number
   numFailedTestSuites: number
   numPassedTests: number
@@ -55,7 +55,7 @@ interface FormattedTestResults {
   numTotalTestSuites: number
   startTime: number
   success: boolean
-  testResults: Array<FormattedTestResult>
+  testResults: Array<JsonTestResult>
   // coverageMap?: CoverageMap | null | undefined
   // numRuntimeErrorTestSuites: number
   // snapshot: SnapshotSummary
@@ -83,7 +83,7 @@ export class JsonReporter implements Reporter {
     const numPassedTests = numTotalTests - numFailedTests
     const numPendingTests = tests.filter(t => t.result?.state === 'run').length
     const numTodoTests = tests.filter(t => t.mode === 'todo').length
-    const testResults: Array<FormattedTestResult> = []
+    const testResults: Array<JsonTestResult> = []
 
     const success = numFailedTestSuites === 0 && numFailedTests === 0
 
@@ -111,7 +111,7 @@ export class JsonReporter implements Reporter {
           duration: t.result?.duration,
           failureMessages: t.result?.errors?.map(e => e.message) || [],
           location: await this.getFailureLocation(t),
-        } as FormattedAssertionResult
+        } as JsonAssertionResult
       }))
 
       if (tests.some(t => t.result?.state === 'run')) {
@@ -120,20 +120,21 @@ export class JsonReporter implements Reporter {
         + 'Please report it to https://github.com/vitest-dev/vitest/issues')
       }
 
+      const hasFailedTests = tests.some(t => t.result?.state === 'fail')
+
       testResults.push({
         assertionResults,
         startTime,
         endTime,
-        status: tests.some(t =>
-          t.result?.state === 'fail')
+        status: file.result?.state === 'fail' || hasFailedTests
           ? 'failed'
           : 'passed',
         message: file.result?.errors?.[0]?.message ?? '',
         name: file.filepath,
       })
     }
 
-    const result: FormattedTestResults = {
+    const result: JsonTestResults = {
       numTotalTestSuites,
       numPassedTestSuites,
       numFailedTestSuites,

test/reporters/fixtures/json-fail-import.test.ts

@@ -0,0 +1 @@
+throw new Error('failed test suite')

test/reporters/tests/html.test.ts

@@ -40,7 +40,7 @@ describe('html reporter', async () => {
   }, 120000)
 
   it('resolves to "failing" status for test file "json-fail"', async () => {
-    const [expected, testFile, basePath] = ['failing', 'json-fail', 'html/fail']
+    const [expected, testFile, basePath] = ['failing', 'json-fail.test', 'html/fail']
 
     await runVitest({ reporters: 'html', outputFile: `${basePath}/index.html`, root }, [testFile])
 

test/reporters/tests/json.test.ts

@@ -11,10 +11,18 @@ describe('json reporter', async () => {
 
     const data = JSON.parse(stdout)
 
-    expect(data.testResults).toHaveLength(1)
-    expect(data.testResults[0].assertionResults).toHaveLength(1)
+    expect(data.testResults).toHaveLength(2)
+
+    const failedImport = data.testResults.find((r: any) => r.name.includes('json-fail-import.test'))!
+    const failedTest = data.testResults.find((r: any) => r.name.includes('json-fail.test'))!
+
+    expect(failedTest.assertionResults).toHaveLength(1)
+    expect(failedImport.assertionResults).toHaveLength(0)
+
+    expect(failedTest.status).toBe('failed')
+    expect(failedImport.status).toBe('failed')
 
-    const result = data.testResults[0].assertionResults[0]
+    const result = failedTest.assertionResults[0]
     delete result.duration
     expect(result).toMatchSnapshot()
   }, 40000)