Skip to content

feat: added param:returnWithLogs for run_tests, count pass/fail with valid tests only #48

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jun 16, 2025
Merged

feat: added param:returnWithLogs for run_tests, count pass/fail with valid tests only #48

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions Editor/Services/ITestRunnerService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,9 @@ public interface ITestRunnerService
/// </summary>
/// <param name="testMode">The test mode to run (EditMode or PlayMode).</param>
/// <param name="returnOnlyFailures">If true, only failed test results are included in the output.</param>
/// <param name="returnWithLogs">If true, all logs are included in the output.</param>
/// <param name="testFilter">A filter string to select specific tests to run.</param>
/// <returns>Task that resolves with test results when tests are complete</returns>
Task<JObject> ExecuteTestsAsync(TestMode testMode, bool returnOnlyFailures, string testFilter);
Task<JObject> ExecuteTestsAsync(TestMode testMode, bool returnOnlyFailures, bool returnWithLogs, string testFilter);
}
}
}
25 changes: 14 additions & 11 deletions Editor/Services/TestRunnerService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ public class TestRunnerService : ITestRunnerService, ICallbacks
private readonly TestRunnerApi _testRunnerApi;
private TaskCompletionSource<JObject> _tcs;
private bool _returnOnlyFailures;
private bool _returnWithLogs;
private List<ITestResultAdaptor> _results;

/// <summary>
Expand Down Expand Up @@ -77,13 +78,15 @@ public async Task<List<ITestAdaptor>> GetAllTestsAsync(string testModeFilter = "
/// </summary>
/// <param name="testMode">The test mode to run (EditMode or PlayMode).</param>
/// <param name="returnOnlyFailures">If true, only failed test results are included in the output.</param>
/// <param name="returnWithLogs">If true, all logs are included in the output.</param>
/// <param name="testFilter">A filter string to select specific tests to run.</param>
/// <returns>Task that resolves with test results when tests are complete</returns>
public async Task<JObject> ExecuteTestsAsync(TestMode testMode, bool returnOnlyFailures, string testFilter = "")
public async Task<JObject> ExecuteTestsAsync(TestMode testMode, bool returnOnlyFailures, bool returnWithLogs, string testFilter = "")
{
_tcs = new TaskCompletionSource<JObject>();
_results = new List<ITestResultAdaptor>();
_returnOnlyFailures = returnOnlyFailures;
_returnWithLogs = returnWithLogs;
var filter = new Filter { testMode = testMode };

if (!string.IsNullOrEmpty(testFilter))
Expand Down Expand Up @@ -191,30 +194,30 @@ private async Task<JObject> WaitForCompletionAsync(int timeoutSeconds)

private JObject BuildResultJson(List<ITestResultAdaptor> results, ITestResultAdaptor result)
{
int pass = results.Count(r => r.ResultState == "Passed");
int fail = results.Count(r => r.ResultState == "Failed");
int skip = results.Count(r => r.ResultState == "Skipped");

var arr = new JArray(results
.Where(r => !r.HasChildren)
.Where(r => !_returnOnlyFailures || r.ResultState == "Failed")
.Select(r => new JObject {
["name"] = r.Name,
["fullName"] = r.FullName,
["state"] = r.ResultState,
["message"] = r.Message,
["duration"] = r.Duration
["duration"] = r.Duration,
["logs"] = _returnWithLogs ? r.Output : null,
["stackTrace"] = r.StackTrace
}));

int testCount = result.PassCount + result.SkipCount + result.FailCount;
return new JObject {
["success"] = true,
["type"] = "text",
["message"] = $"{result.Test.Name} test run completed: {pass}/{results.Count} passed - {fail}/{results.Count} failed - {skip}/{results.Count} skipped",
["message"] = $"{result.Test.Name} test run completed: {result.PassCount}/{testCount} passed - {result.FailCount}/{testCount} failed - {result.SkipCount}/{testCount} skipped",
["resultState"] = result.ResultState,
["durationSeconds"] = result.Duration,
["testCount"] = results.Count,
["passCount"] = pass,
["failCount"] = fail,
["skipCount"] = skip,
["testCount"] = testCount,
["passCount"] = result.PassCount,
["failCount"] = result.FailCount,
["skipCount"] = result.SkipCount,
["results"] = arr
};
}
Expand Down
3 changes: 2 additions & 1 deletion Editor/Tools/RunTestsTool.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ public override async void ExecuteAsync(JObject parameters, TaskCompletionSource
string testModeStr = parameters?["testMode"]?.ToObject<string>() ?? "EditMode";
string testFilter = parameters?["testFilter"]?.ToObject<string>(); // Optional
bool returnOnlyFailures = parameters?["returnOnlyFailures"]?.ToObject<bool>() ?? false; // Optional
bool returnWithLogs = parameters?["returnWithLogs"]?.ToObject<bool>() ?? false; // Optional

TestMode testMode = TestMode.EditMode;

Expand All @@ -48,7 +49,7 @@ public override async void ExecuteAsync(JObject parameters, TaskCompletionSource
McpLogger.LogInfo($"Executing RunTestsTool: Mode={testMode}, Filter={testFilter ?? "(none)"}");

// Call the service to run tests
JObject result = await _testRunnerService.ExecuteTestsAsync(testMode, returnOnlyFailures, testFilter);
JObject result = await _testRunnerService.ExecuteTestsAsync(testMode, returnOnlyFailures, returnWithLogs, testFilter);
tcs.SetResult(result);
}
}
Expand Down
10 changes: 6 additions & 4 deletions Server~/build/tools/runTestsTool.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,9 @@ const toolName = 'run_tests';
const toolDescription = 'Runs Unity\'s Test Runner tests';
const paramsSchema = z.object({
testMode: z.string().optional().default('EditMode').describe('The test mode to run (EditMode or PlayMode) - defaults to EditMode (optional)'),
testFilter: z.string().optional().default('').describe('The specific test filter to run (e.g. specific test name or namespace) (optional)'),
returnOnlyFailures: z.boolean().optional().default(true).describe('Whether to show only failed tests in the results (optional)')
testFilter: z.string().optional().default('').describe('The specific test filter to run (e.g. specific test name or class name, must include namespace) (optional)'),
returnOnlyFailures: z.boolean().optional().default(true).describe('Whether to show only failed tests in the results (optional)'),
returnWithLogs: z.boolean().optional().default(false).describe('Whether to return the test logs in the results (optional)')
});
Comment on lines +8 to 11
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue

Default for returnOnlyFailures diverges from Editor default

Here the schema defaults returnOnlyFailures to true, while the Editor tool falls back to false.
With no explicit user input, the server will always request only failures, changing behaviour unexpectedly after this PR.

-    returnOnlyFailures: z.boolean().optional().default(true)
+    // Keep default aligned with Editor side (false)
+    returnOnlyFailures: z.boolean().optional().default(false)

Committable suggestion skipped: line range outside the PR's diff.

🤖 Prompt for AI Agents 
In Server~/build/tools/runTestsTool.js around lines 8 to 11, the default value
for returnOnlyFailures is set to true, which conflicts with the Editor tool's
default of false. To fix this, change the default value of returnOnlyFailures in
the schema to false so that the server behavior aligns with the Editor tool when
no explicit user input is provided.

/**
* Creates and registers the Run Tests tool with the MCP server
Expand Down Expand Up @@ -41,14 +42,15 @@ export function registerRunTestsTool(server, mcpUnity, logger) {
* @throws McpUnityError if the request to Unity fails
*/
async function toolHandler(mcpUnity, params = {}) {
const { testMode = 'EditMode', testFilter = '', returnOnlyFailures = true } = params;
const { testMode = 'EditMode', testFilter = '', returnOnlyFailures = true, returnWithLogs = false } = params;
// Create and wait for the test run
const response = await mcpUnity.sendRequest({
method: toolName,
params: {
testMode,
testFilter,
returnOnlyFailures
returnOnlyFailures,
returnWithLogs
}
});
// Process the test results
Expand Down
11 changes: 7 additions & 4 deletions Server~/src/tools/runTestsTool.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,9 @@ const toolName = 'run_tests';
const toolDescription = 'Runs Unity\'s Test Runner tests';
const paramsSchema = z.object({
testMode: z.string().optional().default('EditMode').describe('The test mode to run (EditMode or PlayMode) - defaults to EditMode (optional)'),
testFilter: z.string().optional().default('').describe('The specific test filter to run (e.g. specific test name or namespace) (optional)'),
returnOnlyFailures: z.boolean().optional().default(true).describe('Whether to show only failed tests in the results (optional)')
testFilter: z.string().optional().default('').describe('The specific test filter to run (e.g. specific test name or class name, must include namespace) (optional)'),
returnOnlyFailures: z.boolean().optional().default(true).describe('Whether to show only failed tests in the results (optional)'),
returnWithLogs: z.boolean().optional().default(false).describe('Whether to return the test logs in the results (optional)')
});
Comment on lines +13 to 16
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue

Same default mismatch as in JS build artifact

Align the TypeScript source with the Editor default to avoid accidental filtering.

-  returnOnlyFailures: z.boolean().optional().default(true)
+  returnOnlyFailures: z.boolean().optional().default(false)
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
testFilter: z.string().optional().default('').describe('The specific test filter to run (e.g. specific test name or class name, must include namespace) (optional)'),
returnOnlyFailures: z.boolean().optional().default(true).describe('Whether to show only failed tests in the results (optional)'),
returnWithLogs: z.boolean().optional().default(false).describe('Whether to return the test logs in the results (optional)')
});
testFilter: z.string().optional().default('').describe('The specific test filter to run (e.g. specific test name or class name, must include namespace) (optional)'),
- returnOnlyFailures: z.boolean().optional().default(true).describe('Whether to show only failed tests in the results (optional)'),
+ returnOnlyFailures: z.boolean().optional().default(false).describe('Whether to show only failed tests in the results (optional)'),
returnWithLogs: z.boolean().optional().default(false).describe('Whether to return the test logs in the results (optional)')
});
🤖 Prompt for AI Agents 
In Server~/src/tools/runTestsTool.ts around lines 13 to 16, the default value
for returnOnlyFailures is set to true, which mismatches the expected default in
the Editor. Change the default value of returnOnlyFailures from true to false to
align with the Editor default and prevent accidental filtering of test results.


/**
Expand Down Expand Up @@ -56,7 +57,8 @@ async function toolHandler(mcpUnity: McpUnity, params: any = {}): Promise<CallTo
const {
testMode = 'EditMode',
testFilter = '',
returnOnlyFailures = true
returnOnlyFailures = true,
returnWithLogs = false
} = params;

// Create and wait for the test run
Expand All @@ -65,7 +67,8 @@ async function toolHandler(mcpUnity: McpUnity, params: any = {}): Promise<CallTo
params: {
testMode,
testFilter,
returnOnlyFailures
returnOnlyFailures,
returnWithLogs
}
});

Expand Down