add analytics support to the Dart MCP server (#174)

jakemac53 · web-flow · commit 5f9c50fc93c9 · 2025-06-24T11:27:06.000-07:00
Based on dart-lang/tools#2112 Adds analytics events for all tool calls as well as for the runtime errors resource. We could consider a generic resource read event which includes the URI, but we could accidentally create URIs that have some information we don't want to capture, so I chose to instead just customize this and not provide the URI, but a field that describes the type of resource that was read. I did not add any debouncing because I don't think it is necessary, LLMs do not invoke tools in super rapid succession generally, but let me know if you all disagree and think I should do some debouncing.
diff --git a/pkgs/dart_mcp_server/CHANGELOG.md b/pkgs/dart_mcp_server/CHANGELOG.md
@@ -1,4 +1,4 @@
-# Dart SDK 3.8.0 - WP
+# 0.1.0 (Dart SDK 3.8.0) - WP
 
 * Add documentation/homepage/repository links to pub results.
 * Handle relative paths under roots without trailing slashes.
@@ -41,3 +41,4 @@
 * Reduce output size of `run_tests` tool to save on input tokens.
 * Add `--log-file` argument to log all protocol traffic to a file.
 * Improve error text for failed DTD connections as well as the tool description.
+* Add support for injecting an `Analytics` instance to track usage.
diff --git a/pkgs/dart_mcp_server/bin/main.dart b/pkgs/dart_mcp_server/bin/main.dart
@@ -43,6 +43,7 @@ void main(List<String> args) async {
             ),
         forceRootsFallback: parsedArgs.flag(forceRootsFallback),
         sdk: Sdk.find(dartSdkPath: dartSdkPath, flutterSdkPath: flutterSdkPath),
+        analytics: null,
         protocolLogSink: logFileSink,
       )..done.whenComplete(() => logFileSink?.close());
     },
diff --git a/pkgs/dart_mcp_server/lib/src/mixins/dtd.dart b/pkgs/dart_mcp_server/lib/src/mixins/dtd.dart
@@ -10,10 +10,12 @@ import 'package:dds_service_extensions/dds_service_extensions.dart';
 import 'package:dtd/dtd.dart';
 import 'package:json_rpc_2/json_rpc_2.dart';
 import 'package:meta/meta.dart';
+import 'package:unified_analytics/unified_analytics.dart' as ua;
 import 'package:vm_service/vm_service.dart';
 import 'package:vm_service/vm_service_io.dart';
 import 'package:web_socket/web_socket.dart';
 
+import '../utils/analytics.dart';
 import '../utils/constants.dart';
 
 /// Mix this in to any MCPServer to add support for connecting to the Dart
@@ -22,7 +24,8 @@ import '../utils/constants.dart';
 ///
 /// The MCPServer must already have the [ToolsSupport] mixin applied.
 base mixin DartToolingDaemonSupport
-    on ToolsSupport, LoggingSupport, ResourcesSupport {
+    on ToolsSupport, LoggingSupport, ResourcesSupport
+    implements AnalyticsSupport {
   DartToolingDaemon? _dtd;
 
   /// Whether or not the DTD extension to get the active debug sessions is
@@ -115,12 +118,32 @@ base mixin DartToolingDaemonSupport
               '"${debugSession.name}".',
         );
         addResource(resource, (request) async {
-          return ReadResourceResult(
+          final watch = Stopwatch()..start();
+          final result = ReadResourceResult(
             contents: [
               for (var error in errorService.errorLog.errors)
                 TextResourceContents(uri: resource.uri, text: error),
             ],
           );
+          watch.stop();
+          try {
+            analytics?.send(
+              ua.Event.dartMCPEvent(
+                client: clientInfo.name,
+                clientVersion: clientInfo.version,
+                serverVersion: implementation.version,
+                type: AnalyticsEvent.readResource.name,
+                additionalData: ReadResourceMetrics(
+                  kind: ResourceKind.runtimeErrors,
+                  length: result.contents.length,
+                  elapsedMilliseconds: watch.elapsedMilliseconds,
+                ),
+              ),
+            );
+          } catch (e) {
+            log(LoggingLevel.warning, 'Error sending analytics event: $e');
+          }
+          return result;
         });
         errorService.errorsStream.listen((_) => updateResource(resource));
         unawaited(
diff --git a/pkgs/dart_mcp_server/lib/src/server.dart b/pkgs/dart_mcp_server/lib/src/server.dart
@@ -2,18 +2,22 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
+import 'dart:async';
+
 import 'package:dart_mcp/server.dart';
 import 'package:file/file.dart';
 import 'package:file/local.dart';
 import 'package:meta/meta.dart';
 import 'package:process/process.dart';
+import 'package:unified_analytics/unified_analytics.dart';
 
 import 'mixins/analyzer.dart';
 import 'mixins/dash_cli.dart';
 import 'mixins/dtd.dart';
 import 'mixins/pub.dart';
 import 'mixins/pub_dev_search.dart';
 import 'mixins/roots_fallback_support.dart';
+import 'utils/analytics.dart';
 import 'utils/file_system.dart';
 import 'utils/process_manager.dart';
 import 'utils/sdk.dart';
@@ -31,18 +35,23 @@ final class DartMCPServer extends MCPServer
         PubSupport,
         PubDevSupport,
         DartToolingDaemonSupport
-    implements ProcessManagerSupport, FileSystemSupport, SdkSupport {
+    implements
+        AnalyticsSupport,
+        ProcessManagerSupport,
+        FileSystemSupport,
+        SdkSupport {
   DartMCPServer(
     super.channel, {
     required this.sdk,
+    this.analytics,
     @visibleForTesting this.processManager = const LocalProcessManager(),
     @visibleForTesting this.fileSystem = const LocalFileSystem(),
     this.forceRootsFallback = false,
     super.protocolLogSink,
   }) : super.fromStreamChannel(
          implementation: Implementation(
            name: 'dart and flutter tooling',
-           version: '0.1.0-wip',
+           version: '0.1.0',
          ),
          instructions:
              'This server helps to connect Dart and Flutter developers to '
@@ -62,4 +71,50 @@ final class DartMCPServer extends MCPServer
 
   @override
   final Sdk sdk;
+
+  @override
+  final Analytics? analytics;
+
+  @override
+  /// Automatically logs all tool calls via analytics by wrapping the [impl],
+  /// if [analytics] is not `null`.
+  void registerTool(
+    Tool tool,
+    FutureOr<CallToolResult> Function(CallToolRequest) impl,
+  ) {
+    // For type promotion.
+    final analytics = this.analytics;
+
+    super.registerTool(
+      tool,
+      analytics == null
+          ? impl
+          : (CallToolRequest request) async {
+            final watch = Stopwatch()..start();
+            CallToolResult? result;
+            try {
+              return result = await impl(request);
+            } finally {
+              watch.stop();
+              try {
+                analytics.send(
+                  Event.dartMCPEvent(
+                    client: clientInfo.name,
+                    clientVersion: clientInfo.version,
+                    serverVersion: implementation.version,
+                    type: AnalyticsEvent.callTool.name,
+                    additionalData: CallToolMetrics(
+                      tool: request.name,
+                      success: result != null && result.isError != true,
+                      elapsedMilliseconds: watch.elapsedMilliseconds,
+                    ),
+                  ),
+                );
+              } catch (e) {
+                log(LoggingLevel.warning, 'Error sending analytics event: $e');
+              }
+            }
+          },
+    );
+  }
 }
diff --git a/pkgs/dart_mcp_server/lib/src/utils/analytics.dart b/pkgs/dart_mcp_server/lib/src/utils/analytics.dart
@@ -0,0 +1,76 @@
+// Copyright (c) 2025, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'package:unified_analytics/unified_analytics.dart';
+
+/// An interface class that provides a access to an [Analytics] instance, if
+/// enabled.
+///
+/// The `DartMCPServer` class implements this class so that [Analytics]
+/// methods can be easily mocked during testing.
+abstract interface class AnalyticsSupport {
+  Analytics? get analytics;
+}
+
+enum AnalyticsEvent { callTool, readResource }
+
+/// The metrics for a resources/read MCP handler.
+final class ReadResourceMetrics extends CustomMetrics {
+  /// The kind of resource that was read.
+  ///
+  /// We don't want to record the full URI.
+  final ResourceKind kind;
+
+  /// The length of the resource.
+  final int length;
+
+  /// The time it took to read the resource.
+  final int elapsedMilliseconds;
+
+  ReadResourceMetrics({
+    required this.kind,
+    required this.length,
+    required this.elapsedMilliseconds,
+  });
+
+  @override
+  Map<String, Object> toMap() => {
+    _kind: kind.name,
+    _length: length,
+    _elapsedMilliseconds: elapsedMilliseconds,
+  };
+}
+
+/// The metrics for a tools/call MCP handler.
+final class CallToolMetrics extends CustomMetrics {
+  /// The name of the tool that was invoked.
+  final String tool;
+
+  /// Whether or not the tool call succeeded.
+  final bool success;
+
+  /// The time it took to invoke the tool.
+  final int elapsedMilliseconds;
+
+  CallToolMetrics({
+    required this.tool,
+    required this.success,
+    required this.elapsedMilliseconds,
+  });
+
+  @override
+  Map<String, Object> toMap() => {
+    _tool: tool,
+    _success: success,
+    _elapsedMilliseconds: elapsedMilliseconds,
+  };
+}
+
+enum ResourceKind { runtimeErrors }
+
+const _elapsedMilliseconds = 'elapsedMilliseconds';
+const _kind = 'kind';
+const _length = 'length';
+const _success = 'success';
+const _tool = 'tool';
diff --git a/pkgs/dart_mcp_server/pubspec.yaml b/pkgs/dart_mcp_server/pubspec.yaml
@@ -2,7 +2,6 @@ name: dart_mcp_server
 description: >-
   An MCP server for Dart projects, exposing various developer tools to AI
   models.
-
 publish_to: none
 
 environment:
@@ -33,6 +32,7 @@ dependencies:
   pool: ^1.5.1
   process: ^5.0.3
   stream_channel: ^2.1.4
+  unified_analytics: ^8.0.2
   vm_service: ^15.0.0
   watcher: ^1.1.1
   web_socket: ^1.0.1
diff --git a/pkgs/dart_mcp_server/test/dart_tooling_mcp_server_test.dart b/pkgs/dart_mcp_server/test/dart_tooling_mcp_server_test.dart
@@ -5,12 +5,94 @@
 import 'dart:async';
 import 'dart:io';
 
+import 'package:dart_mcp/server.dart';
+import 'package:dart_mcp_server/src/server.dart';
 import 'package:test/test.dart';
 import 'package:test_descriptor/test_descriptor.dart' as d;
+import 'package:unified_analytics/testing.dart';
+import 'package:unified_analytics/unified_analytics.dart';
 
 import 'test_harness.dart';
 
 void main() {
+  group('analytics', () {
+    late TestHarness testHarness;
+    late DartMCPServer server;
+    late FakeAnalytics analytics;
+
+    setUp(() async {
+      testHarness = await TestHarness.start(inProcess: true);
+      server = testHarness.serverConnectionPair.server!;
+      analytics = server.analytics as FakeAnalytics;
+    });
+
+    test('sends analytics for successful tool calls', () async {
+      server.registerTool(
+        Tool(name: 'hello', inputSchema: Schema.object()),
+        (_) => CallToolResult(content: [Content.text(text: 'world')]),
+      );
+      final result = await testHarness.callToolWithRetry(
+        CallToolRequest(name: 'hello'),
+      );
+      expect((result.content.single as TextContent).text, 'world');
+      expect(
+        analytics.sentEvents.single,
+        isA<Event>()
+            .having((e) => e.eventName, 'eventName', DashEvent.dartMCPEvent)
+            .having(
+              (e) => e.eventData,
+              'eventData',
+              equals({
+                'client': server.clientInfo.name,
+                'clientVersion': server.clientInfo.version,
+                'serverVersion': server.implementation.version,
+                'type': 'callTool',
+                'tool': 'hello',
+                'success': true,
+                'elapsedMilliseconds': isA<int>(),
+              }),
+            ),
+      );
+    });
+
+    test('sends analytics for failed tool calls', () async {
+      server.registerTool(
+        Tool(name: 'hello', inputSchema: Schema.object()),
+        (_) => CallToolResult(isError: true, content: []),
+      );
+      final result = await testHarness.mcpServerConnection.callTool(
+        CallToolRequest(name: 'hello'),
+      );
+      expect(result.isError, true);
+      expect(
+        analytics.sentEvents.single,
+        isA<Event>()
+            .having((e) => e.eventName, 'eventName', DashEvent.dartMCPEvent)
+            .having(
+              (e) => e.eventData,
+              'eventData',
+              equals({
+                'client': server.clientInfo.name,
+                'clientVersion': server.clientInfo.version,
+                'serverVersion': server.implementation.version,
+                'type': 'callTool',
+                'tool': 'hello',
+                'success': false,
+                'elapsedMilliseconds': isA<int>(),
+              }),
+            ),
+      );
+    });
+
+    test('Changelog version matches dart server version', () {
+      final changelogFile = File('CHANGELOG.md');
+      expect(
+        changelogFile.readAsLinesSync().first.split(' ')[1],
+        testHarness.serverConnectionPair.server!.implementation.version,
+      );
+    });
+  });
+
   group('--log-file', () {
     late d.FileDescriptor logDescriptor;
     late TestHarness testHarness;
@@ -41,7 +123,7 @@ void main() {
 
       // Wait for the process to release the file.
       await doWithRetries(() => File(logDescriptor.io.path).delete());
-    });
+    }, skip: 'https://github.com/dart-lang/ai/issues/181');
   });
 }
 
diff --git a/pkgs/dart_mcp_server/test/test_harness.dart b/pkgs/dart_mcp_server/test/test_harness.dart
diff --git a/pkgs/dart_mcp_server/test/tools/dtd_test.dart b/pkgs/dart_mcp_server/test/tools/dtd_test.dart
