diff --git a/.gitignore b/.gitignore index d6e75923ad..9763042250 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,4 @@ vcpkg_installed/ # Temporary folder to refresh SDK with TypeSpec. TempTypeSpecFiles/ +test-results/ diff --git a/eng/scripts/Convert-TestResultsToJUnit.ps1 b/eng/scripts/Convert-TestResultsToJUnit.ps1 new file mode 100644 index 0000000000..d667923cbd --- /dev/null +++ b/eng/scripts/Convert-TestResultsToJUnit.ps1 @@ -0,0 +1,125 @@ +#!/usr/bin/env pwsh + +#Requires -Version 7.0 +<# +.SYNOPSIS +Converts cargo test JSON output to JUnit XML format using cargo2junit. + +.DESCRIPTION +This script converts the JSON output files from cargo test (captured by Test-Packages.ps1 in CI mode) +to JUnit XML format suitable for publishing to Azure DevOps test results using the cargo2junit tool. + +.PARAMETER TestResultsDirectory +The directory containing JSON test result files. Defaults to test-results in the repo root. + +.PARAMETER OutputDirectory +The directory where JUnit XML files should be written. Defaults to test-results/junit in the repo root. + +.EXAMPLE +./eng/scripts/Convert-TestResultsToJUnit.ps1 + +.EXAMPLE +./eng/scripts/Convert-TestResultsToJUnit.ps1 -TestResultsDirectory ./test-results -OutputDirectory ./junit-results +#> + +param( + [string]$TestResultsDirectory, + [string]$OutputDirectory +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version 2.0 + +# Get repo root +$RepoRoot = Resolve-Path (Join-Path $PSScriptRoot .. ..) + +# Set default directories +if (!$TestResultsDirectory) { + $TestResultsDirectory = Join-Path $RepoRoot "test-results" +} + +if (!$OutputDirectory) { + $OutputDirectory = Join-Path $RepoRoot "test-results" "junit" +} + +Write-Host "Converting test results from JSON to JUnit XML using cargo2junit" +Write-Host " Input directory: $TestResultsDirectory" +Write-Host " Output directory: $OutputDirectory" + +# Check if test results directory exists +if (!(Test-Path $TestResultsDirectory)) { + Write-Warning "Test results directory not found: $TestResultsDirectory" + Write-Host "No test results to convert." + exit 0 +} + +# Create output directory if it doesn't exist +if (!(Test-Path $OutputDirectory)) { + New-Item -ItemType Directory -Path $OutputDirectory | Out-Null + Write-Host "Created output directory: $OutputDirectory" +} + +# Check if cargo2junit is installed +$cargo2junitPath = Get-Command cargo2junit -ErrorAction SilentlyContinue +if (!$cargo2junitPath) { + Write-Host "cargo2junit not found. Installing..." + cargo install cargo2junit + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to install cargo2junit" + exit 1 + } + Write-Host "cargo2junit installed successfully" +} + +# Get all JSON files in the test results directory +$jsonFiles = @(Get-ChildItem -Path $TestResultsDirectory -Filter "*.json" -File) + +if ($jsonFiles.Count -eq 0) { + Write-Warning "No JSON files found in $TestResultsDirectory" + Write-Host "No test results to convert." + exit 0 +} + +Write-Host "`nConverting $($jsonFiles.Count) JSON file(s) to JUnit XML..." + +$convertedCount = 0 +$failedCount = 0 + +foreach ($jsonFile in $jsonFiles) { + $baseName = [System.IO.Path]::GetFileNameWithoutExtension($jsonFile.Name) + $junitFile = Join-Path $OutputDirectory "$baseName.xml" + + Write-Host " Converting: $($jsonFile.Name) -> $([System.IO.Path]::GetFileName($junitFile))" + + try { + # Convert JSON to JUnit XML using cargo2junit + Get-Content $jsonFile.FullName | cargo2junit > $junitFile + + if ($LASTEXITCODE -ne 0) { + Write-Warning " cargo2junit returned exit code $LASTEXITCODE for $($jsonFile.Name)" + $failedCount++ + } + else { + $convertedCount++ + } + } + catch { + Write-Warning " Failed to convert $($jsonFile.Name): $_" + $failedCount++ + } +} + +Write-Host "`nConversion complete:" +Write-Host " Successfully converted: $convertedCount" +if ($failedCount -gt 0) { + Write-Host " Failed to convert: $failedCount" -ForegroundColor Yellow +} + +Write-Host "`nJUnit XML files are available in: $OutputDirectory" + +# Exit with error if any conversions failed +if ($failedCount -gt 0) { + exit 1 +} + +exit 0 diff --git a/eng/scripts/TEST-RESULTS-README.md b/eng/scripts/TEST-RESULTS-README.md new file mode 100644 index 0000000000..52a2a210d6 --- /dev/null +++ b/eng/scripts/TEST-RESULTS-README.md @@ -0,0 +1,150 @@ +# Test Results Reporting + +This directory contains scripts for capturing cargo test results and converting them to JUnit XML format for Azure DevOps. + +## Overview + +The test results reporting uses: +1. **Nightly Rust's native JSON test output** (`cargo +nightly test -- --format json -Z unstable-options`) +2. **cargo2junit** tool to convert JSON to JUnit XML + +## Scripts + +### Test-Packages.ps1 + +Enhanced to support CI mode with `-CI` switch parameter. + +**CI Mode (`-CI` flag):** +- Uses `cargo +nightly test -- --format json -Z unstable-options` +- Captures JSON output to uniquely named files in `test-results/` directory +- Parses JSON and displays human-readable summaries +- Shows pass/fail/ignored counts and lists failed tests + +**Standard Mode (no `-CI` flag):** +- Original behavior using `Invoke-LoggedCommand` +- Human-readable output directly to console + +**Usage:** +```powershell +# CI mode +./eng/scripts/Test-Packages.ps1 -PackageInfoDirectory ./PackageInfo -CI + +# Standard mode +./eng/scripts/Test-Packages.ps1 -PackageInfoDirectory ./PackageInfo +``` + +### Convert-TestResultsToJUnit.ps1 + +Converts JSON test results to JUnit XML format using cargo2junit. + +**Features:** +- Automatically installs cargo2junit if not present +- Processes all JSON files in test-results directory +- Outputs JUnit XML to test-results/junit directory +- Compatible with Azure DevOps PublishTestResults task + +**Usage:** +```powershell +./eng/scripts/Convert-TestResultsToJUnit.ps1 + +# Or with custom directories +./eng/scripts/Convert-TestResultsToJUnit.ps1 -TestResultsDirectory ./test-results -OutputDirectory ./junit +``` + +## Pipeline Integration + +Example Azure DevOps pipeline YAML: + +```yaml +# Run tests with JSON output capture +- task: Powershell@2 + displayName: "Test Packages" + inputs: + pwsh: true + filePath: $(Build.SourcesDirectory)/eng/scripts/Test-Packages.ps1 + arguments: > + -PackageInfoDirectory '$(Build.ArtifactStagingDirectory)/PackageInfo' + -CI + +# Convert JSON to JUnit XML +- task: Powershell@2 + displayName: "Convert Test Results to JUnit XML" + condition: succeededOrFailed() + inputs: + pwsh: true + filePath: $(Build.SourcesDirectory)/eng/scripts/Convert-TestResultsToJUnit.ps1 + +# Publish test results to Azure DevOps +- task: PublishTestResults@2 + displayName: "Publish Test Results" + condition: succeededOrFailed() + inputs: + testResultsFormat: 'JUnit' + testResultsFiles: '**/test-results/junit/*.xml' + testRunTitle: 'Rust Tests' + mergeTestResults: true + failTaskOnFailedTests: false +``` + +## Requirements + +- **PowerShell 7.0+** (already required by existing scripts) +- **Nightly Rust toolchain** (installed automatically by rustup when using `cargo +nightly`) +- **cargo2junit** (installed automatically by Convert-TestResultsToJUnit.ps1 if needed) + +## Test Results Format + +### Directory Structure +``` +test-results/ +├── {package}-doctest-{timestamp}.json # JSON test output from doc tests +├── {package}-alltargets-{timestamp}.json # JSON test output from all-targets tests +└── junit/ + ├── {package}-doctest-{timestamp}.xml # JUnit XML for doc tests + └── {package}-alltargets-{timestamp}.xml # JUnit XML for all-targets tests +``` + +### JSON Format + +Nightly Rust outputs newline-delimited JSON with events like: +```json +{ "type": "test", "event": "started", "name": "test_name" } +{ "type": "test", "name": "test_name", "event": "ok" } +{ "type": "suite", "event": "ok", "passed": 30, "failed": 0, "ignored": 0 } +``` + +### JUnit XML Format + +cargo2junit converts to standard JUnit XML: +```xml + + + + + +``` + +## Troubleshooting + +### Nightly Rust not installed +If you see errors about nightly not being available: +```bash +rustup toolchain install nightly +``` + +### cargo2junit not found +The Convert-TestResultsToJUnit.ps1 script automatically installs it, but you can manually install: +```bash +cargo install cargo2junit +``` + +### No test results generated +Make sure the `-CI` flag is passed to Test-Packages.ps1 when running in CI mode. + +## Benefits of This Approach + +1. **Native Format**: Uses Rust's native JSON test output format (no custom parsing) +2. **Reliable**: cargo2junit is purpose-built for this conversion +3. **Simple**: Minimal code, leverages existing tools +4. **Maintainable**: Less custom code to maintain +5. **Feature-Rich**: Gets full test metadata from Rust's test harness diff --git a/eng/scripts/Test-Packages.ps1 b/eng/scripts/Test-Packages.ps1 index a9515d9f8d..dc648f272b 100755 --- a/eng/scripts/Test-Packages.ps1 +++ b/eng/scripts/Test-Packages.ps1 @@ -2,16 +2,78 @@ #Requires -Version 7.0 param( - [string]$PackageInfoDirectory + [string]$PackageInfoDirectory, + [switch]$CI ) $ErrorActionPreference = 'Stop' Set-StrictMode -Version 2.0 . "$PSScriptRoot/../common/scripts/common.ps1" +# Helper function to parse test results from JSON and output human-readable summary +function Write-TestSummary { + param( + [string]$JsonFile, + [string]$PackageName + ) + + if (!(Test-Path $JsonFile)) { + Write-Warning "Test results file not found: $JsonFile" + return + } + + $passed = 0 + $failed = 0 + $ignored = 0 + $failedTests = @() + + # Parse JSON output (newline-delimited JSON) + Get-Content $JsonFile | ForEach-Object { + try { + $event = $_ | ConvertFrom-Json -ErrorAction SilentlyContinue + if ($event.type -eq "test" -and $event.event) { + switch ($event.event) { + "ok" { $passed++ } + "failed" { + $failed++ + $failedTests += $event.name + } + "ignored" { $ignored++ } + } + } + } + catch { + # Ignore lines that aren't valid JSON + } + } + + Write-Host "`n========================================" -ForegroundColor Cyan + Write-Host "Test Summary: $PackageName" -ForegroundColor Cyan + Write-Host "========================================" -ForegroundColor Cyan + Write-Host "Passed: $passed" -ForegroundColor Green + Write-Host "Failed: $failed" -ForegroundColor $(if ($failed -gt 0) { "Red" } else { "Green" }) + Write-Host "Ignored: $ignored" -ForegroundColor Yellow + + if ($failed -gt 0) { + Write-Host "`nFailed tests:" -ForegroundColor Red + foreach ($test in $failedTests) { + Write-Host " - $test" -ForegroundColor Red + } + Write-Host "`nℹ️ Additional details are available in the test tab for the build." -ForegroundColor Yellow + } + Write-Host "========================================`n" -ForegroundColor Cyan + + return @{ + Passed = $passed + Failed = $failed + Ignored = $ignored + } +} + Write-Host @" Testing packages with PackageInfoDirectory: '$PackageInfoDirectory' + CI Mode: $CI RUSTFLAGS: '$env:RUSTFLAGS' RUSTDOCFLAGS: '$env:RUSTDOCFLAGS' RUST_LOG: '$env:RUST_LOG' @@ -20,6 +82,16 @@ Testing packages with ARM_OIDC_TOKEN: $($env:ARM_OIDC_TOKEN ? 'present' : 'not present') "@ +# Create directory for test results if in CI mode +$testResultsDir = $null +if ($CI) { + $testResultsDir = Join-Path $RepoRoot "test-results" + if (!(Test-Path $testResultsDir)) { + New-Item -ItemType Directory -Path $testResultsDir | Out-Null + } + Write-Host "Test results will be saved to: $testResultsDir" +} + if ($PackageInfoDirectory) { if (!(Test-Path $PackageInfoDirectory)) { Write-Error "Package info path '$PackageInfoDirectory' does not exist." @@ -39,6 +111,9 @@ foreach ($package in $packagesToTest) { Write-Host " '$($package.Name)' in '$($package.DirectoryPath)'" } +$allTestResults = @() +$hasFailures = $false + foreach ($package in $packagesToTest) { Push-Location ([System.IO.Path]::Combine($RepoRoot, $package.DirectoryPath)) try { @@ -59,10 +134,58 @@ foreach ($package in $packagesToTest) { Invoke-LoggedCommand "cargo build --keep-going" -GroupOutput Write-Host "`n`n" - Invoke-LoggedCommand "cargo test --doc --no-fail-fast" -GroupOutput + # Generate unique filenames for test outputs if in CI mode + $timestamp = Get-Date -Format "yyyyMMdd-HHmmss-fff" + $sanitizedPackageName = $package.Name -replace '[^a-zA-Z0-9_-]', '_' + + # Run doc tests + if ($CI) { + $docTestOutput = Join-Path $testResultsDir "$sanitizedPackageName-doctest-$timestamp.json" + Write-Host "Running doc tests with JSON output to: $docTestOutput" + + # Use cargo +nightly test with --format json and -Z unstable-options + $output = & cargo +nightly test --doc --no-fail-fast -- --format json -Z unstable-options 2>&1 + $exitCode = $LASTEXITCODE + + # Write JSON output to file + $output | Out-File -FilePath $docTestOutput -Encoding utf8 + + # Also display the output + $output | ForEach-Object { Write-Host $_ } + + # Parse and display summary + $docResults = Write-TestSummary -JsonFile $docTestOutput -PackageName "$($package.Name) (doc tests)" + if ($exitCode -ne 0) { $hasFailures = $true } + $allTestResults += @{ Package = $package.Name; Type = "doc"; Results = $docResults } + } + else { + Invoke-LoggedCommand "cargo test --doc --no-fail-fast" -GroupOutput + } Write-Host "`n`n" - Invoke-LoggedCommand "cargo test --all-targets --no-fail-fast" -GroupOutput + # Run all-targets tests + if ($CI) { + $allTargetsOutput = Join-Path $testResultsDir "$sanitizedPackageName-alltargets-$timestamp.json" + Write-Host "Running all-targets tests with JSON output to: $allTargetsOutput" + + # Use cargo +nightly test with --format json and -Z unstable-options + $output = & cargo +nightly test --all-targets --no-fail-fast -- --format json -Z unstable-options 2>&1 + $exitCode = $LASTEXITCODE + + # Write JSON output to file + $output | Out-File -FilePath $allTargetsOutput -Encoding utf8 + + # Also display the output + $output | ForEach-Object { Write-Host $_ } + + # Parse and display summary + $allTargetsResults = Write-TestSummary -JsonFile $allTargetsOutput -PackageName "$($package.Name) (all targets)" + if ($exitCode -ne 0) { $hasFailures = $true } + $allTestResults += @{ Package = $package.Name; Type = "all-targets"; Results = $allTargetsResults } + } + else { + Invoke-LoggedCommand "cargo test --all-targets --no-fail-fast" -GroupOutput + } Write-Host "`n`n" $cleanupScript = Join-Path $packageDirectory "Test-Cleanup.ps1" @@ -76,3 +199,38 @@ foreach ($package in $packagesToTest) { Pop-Location } } + +# Print overall summary if in CI mode +if ($CI -and $allTestResults.Count -gt 0) { + Write-Host "`n`n" + Write-Host "========================================" -ForegroundColor Cyan + Write-Host "OVERALL TEST SUMMARY" -ForegroundColor Cyan + Write-Host "========================================" -ForegroundColor Cyan + + $totalPassed = 0 + $totalFailed = 0 + $totalIgnored = 0 + + foreach ($result in $allTestResults) { + if ($result.Results) { + $totalPassed += $result.Results.Passed + $totalFailed += $result.Results.Failed + $totalIgnored += $result.Results.Ignored + } + } + + Write-Host "Total Passed: $totalPassed" -ForegroundColor Green + Write-Host "Total Failed: $totalFailed" -ForegroundColor $(if ($totalFailed -gt 0) { "Red" } else { "Green" }) + Write-Host "Total Ignored: $totalIgnored" -ForegroundColor Yellow + + if ($totalFailed -gt 0) { + Write-Host "`nℹ️ Additional details are available in the test tab for the build." -ForegroundColor Yellow + } + + Write-Host "========================================`n" -ForegroundColor Cyan + + # Exit with error if there were failures + if ($hasFailures) { + exit 1 + } +}