Add JSON schema for test runner arguments (#169)

Add strong language to always use the `run_tests` tool.

Add a JSON schema for the CLI arguments to the test runner. The schema
was dumped using dart-lang/core#897

Add a test for the conversion of boolean, string, and list of string
arguments, as well as overriding the reporter argument.

Author: natebosch

pkgs/dart_mcp_server/lib/src/mixins/dash_cli.dart

@@ -3,6 +3,7 @@
 // BSD-style license that can be found in the LICENSE file.
 
 import 'dart:async';
+import 'dart:convert';
 
 import 'package:dart_mcp/server.dart';
 import 'package:path/path.dart' as p;
@@ -66,9 +67,18 @@ base mixin DashCliSupport on ToolsSupport, LoggingSupport, RootsTrackingSupport
 
   /// Implementation of the [runTestsTool].
   Future<CallToolResult> _runTests(CallToolRequest request) async {
+    final testRunnerArguments =
+        request.arguments?[ParameterNames.testRunnerArgs]
+            as Map<String, Object?>?;
+    final hasReporterArg =
+        testRunnerArguments?.containsKey('reporter') ?? false;
     return runCommandInRoots(
       request,
-      arguments: ['test', '--reporter=failures-only'],
+      arguments: [
+        'test',
+        if (!hasReporterArg) '--reporter=failures-only',
+        ...?testRunnerArguments?.asCliArgs(),
+      ],
       commandDescription: 'dart|flutter test',
       processManager: processManager,
       knownRoots: await roots,
@@ -187,14 +197,26 @@ base mixin DashCliSupport on ToolsSupport, LoggingSupport, RootsTrackingSupport
     ),
   );
 
-  static final runTestsTool = Tool(
-    name: 'run_tests',
-    description: 'Runs Dart or Flutter tests for the given project roots.',
-    annotations: ToolAnnotations(title: 'Run tests', readOnlyHint: true),
-    inputSchema: Schema.object(
-      properties: {ParameterNames.roots: rootsSchema(supportsPaths: true)},
-    ),
-  );
+  static final Tool runTestsTool = () {
+    final cliSchemaJson =
+        jsonDecode(_dartTestCliSchema) as Map<String, Object?>;
+    const blocklist = {'color', 'debug', 'help', 'pause-after-load', 'version'};
+    cliSchemaJson.removeWhere((argument, _) => blocklist.contains(argument));
+    final cliSchema = Schema.fromMap(cliSchemaJson);
+    return Tool(
+      name: 'run_tests',
+      description:
+          'Run Dart or Flutter tests with an agent centric UX. '
+          'ALWAYS use instead of `dart test` or `flutter test` shell commands.',
+      annotations: ToolAnnotations(title: 'Run tests', readOnlyHint: true),
+      inputSchema: Schema.object(
+        properties: {
+          ParameterNames.roots: rootsSchema(supportsPaths: true),
+          ParameterNames.testRunnerArgs: cliSchema,
+        },
+      ),
+    );
+  }();
 
   static final createProjectTool = Tool(
     name: 'create_project',
@@ -245,3 +267,29 @@ base mixin DashCliSupport on ToolsSupport, LoggingSupport, RootsTrackingSupport
     'ios',
   };
 }
+
+extension on Map<String, Object?> {
+  Iterable<String> asCliArgs() sync* {
+    for (final MapEntry(:key, :value) in entries) {
+      if (value is List) {
+        for (final element in value) {
+          yield '--$key';
+          yield element as String;
+        }
+        continue;
+      }
+      yield '--$key';
+      if (value is bool) continue;
+      yield value as String;
+    }
+  }
+}
+
+// Generated by the test runner using an un-merged commit.
+// To update merge the latest argument changes to the `json-schema` branch and
+// run with the `--json-help` argument. Pipe to `sed 's/\\/\\\\/g'` to escape
+// as a Dart source string.
+// https://github.com/dart-lang/test/pull/2508
+const _dartTestCliSchema = '''
+{"type":"object","properties":{"help":{"type":"boolean","description":"Show this usage information.\\ndefaults to \\"false\\""},"version":{"type":"boolean","description":"Show the package:test version.\\ndefaults to \\"false\\""},"name":{"type":"array","description":"A substring of the name of the test to run.\\nRegular expression syntax is supported.\\nIf passed multiple times, tests must match all substrings.\\ndefaults to \\"[]\\"","items":{"type":"string"}},"plain-name":{"type":"array","description":"A plain-text substring of the name of the test to run.\\nIf passed multiple times, tests must match all substrings.\\ndefaults to \\"[]\\"","items":{"type":"string"}},"tags":{"type":"array","description":"Run only tests with all of the specified tags.\\nSupports boolean selector syntax.\\ndefaults to \\"[]\\"","items":{"type":"string"}},"exclude-tags":{"type":"array","description":"Don't run tests with any of the specified tags.\\nSupports boolean selector syntax.\\ndefaults to \\"[]\\"","items":{"type":"string"}},"run-skipped":{"type":"boolean","description":"Run skipped tests instead of skipping them.\\ndefaults to \\"false\\""},"platform":{"type":"array","description":"The platform(s) on which to run the tests.\\n[vm (default), chrome, firefox, edge, node].\\nEach platform supports the following compilers:\\n[vm]: kernel (default), source, exe\\n[chrome]: dart2js (default), dart2wasm\\n[firefox]: dart2js (default), dart2wasm\\n[edge]: dart2js (default)\\n[node]: dart2js (default), dart2wasm\\ndefaults to \\"[]\\"","items":{"type":"string"}},"compiler":{"type":"array","description":"The compiler(s) to use to run tests, supported compilers are [dart2js, dart2wasm, exe, kernel, source].\\nEach platform has a default compiler but may support other compilers.\\nYou can target a compiler to a specific platform using arguments of the following form [<platform-selector>:]<compiler>.\\nIf a platform is specified but no given compiler is supported for that platform, then it will use its default compiler.\\ndefaults to \\"[]\\"","items":{"type":"string"}},"preset":{"type":"array","description":"The configuration preset(s) to use.\\ndefaults to \\"[]\\"","items":{"type":"string"}},"concurrency":{"type":"string","description":"The number of concurrent test suites run.\\ndefaults to \\"8\\""},"total-shards":{"type":"string","description":"The total number of invocations of the test runner being run."},"shard-index":{"type":"string","description":"The index of this test runner invocation (of --total-shards)."},"timeout":{"type":"string","description":"The default test timeout. For example: 15s, 2x, none\\ndefaults to \\"30s\\""},"ignore-timeouts":{"type":"boolean","description":"Ignore all timeouts (useful if debugging)\\ndefaults to \\"false\\""},"pause-after-load":{"type":"boolean","description":"Pause for debugging before any tests execute.\\nImplies --concurrency=1, --debug, and --ignore-timeouts.\\nCurrently only supported for browser tests.\\ndefaults to \\"false\\""},"debug":{"type":"boolean","description":"Run the VM and Chrome tests in debug mode.\\ndefaults to \\"false\\""},"coverage":{"type":"string","description":"Gather coverage and output it to the specified directory.\\nImplies --debug."},"chain-stack-traces":{"type":"boolean","description":"Use chained stack traces to provide greater exception details\\nespecially for asynchronous code. It may be useful to disable\\nto provide improved test performance but at the cost of\\ndebuggability.\\ndefaults to \\"false\\""},"no-retry":{"type":"boolean","description":"Don't rerun tests that have retry set.\\ndefaults to \\"false\\""},"test-randomize-ordering-seed":{"type":"string","description":"Use the specified seed to randomize the execution order of test cases.\\nMust be a 32bit unsigned integer or \\"random\\".\\nIf \\"random\\", pick a random seed to use.\\nIf not passed, do not randomize test case execution order."},"fail-fast":{"type":"boolean","description":"Stop running tests after the first failure.\\n\\ndefaults to \\"false\\""},"reporter":{"type":"string","description":"Set how to print test results.\\ndefaults to \\"compact\\"\\nallowed values: compact, expanded, failures-only, github, json, silent"},"file-reporter":{"type":"string","description":"Enable an additional reporter writing test results to a file.\\nShould be in the form <reporter>:<filepath>, Example: \\"json:reports/tests.json\\""},"verbose-trace":{"type":"boolean","description":"Emit stack traces with core library frames.\\ndefaults to \\"false\\""},"js-trace":{"type":"boolean","description":"Emit raw JavaScript stack traces for browser tests.\\ndefaults to \\"false\\""},"color":{"type":"boolean","description":"Use terminal colors.\\n(auto-detected by default)\\ndefaults to \\"false\\""}},"required":[]}
+''';

pkgs/dart_mcp_server/lib/src/utils/constants.dart

@@ -21,6 +21,7 @@ extension ParameterNames on Never {
   static const root = 'root';
   static const roots = 'roots';
   static const template = 'template';
+  static const testRunnerArgs = 'testRunnerArgs';
   static const uri = 'uri';
   static const uris = 'uris';
 }

pkgs/dart_mcp_server/test/tools/dart_cli_test.dart

@@ -184,6 +184,69 @@ dependencies:
       ]);
     });
 
+    test('flutter and dart package tests with extra arguments', () async {
+      testHarness.mcpClient.addRoot(dartCliAppRoot);
+      testHarness.mcpClient.addRoot(exampleFlutterAppRoot);
+      await pumpEventQueue();
+      final request = CallToolRequest(
+        name: DashCliSupport.runTestsTool.name,
+        arguments: {
+          ParameterNames.testRunnerArgs: {
+            'run-skipped': true,
+            'platform': ['vm', 'chrome'],
+            'reporter': 'json',
+          },
+          ParameterNames.roots: [
+            {
+              ParameterNames.root: exampleFlutterAppRoot.uri,
+              ParameterNames.paths: ['foo_test.dart', 'bar_test.dart'],
+            },
+            {
+              ParameterNames.root: dartCliAppRoot.uri,
+              ParameterNames.paths: ['zip_test.dart'],
+            },
+          ],
+        },
+      );
+      final result = await testHarness.callToolWithRetry(request);
+
+      // Verify the command was sent to the process manager without error.
+      expect(result.isError, isNot(true));
+      expect(testProcessManager.commandsRan, [
+        equalsCommand((
+          command: [
+            endsWith(flutterExecutableName),
+            'test',
+            '--run-skipped',
+            '--platform',
+            'vm',
+            '--platform',
+            'chrome',
+            '--reporter',
+            'json',
+            'foo_test.dart',
+            'bar_test.dart',
+          ],
+          workingDirectory: exampleFlutterAppRoot.path,
+        )),
+        equalsCommand((
+          command: [
+            endsWith(dartExecutableName),
+            'test',
+            '--run-skipped',
+            '--platform',
+            'vm',
+            '--platform',
+            'chrome',
+            '--reporter',
+            'json',
+            'zip_test.dart',
+          ],
+          workingDirectory: dartCliAppRoot.path,
+        )),
+      ]);
+    });
+
     group('create', () {
       test('creates a Dart project', () async {
         testHarness.mcpClient.addRoot(dartCliAppRoot);