Skip to content

add analytics tracking for prompts #246

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jul 28, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 45 additions & 0 deletions pkgs/dart_mcp_server/lib/src/server.dart
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,51 @@ final class DartMCPServer extends MCPServer
validateArguments: validateArguments,
);
}

@override
void addPrompt(
Prompt prompt,
FutureOr<GetPromptResult> Function(GetPromptRequest) impl,
) {
// For type promotion.
final analytics = this.analytics;

super.addPrompt(
prompt,
analytics == null
? impl
: (request) async {
final watch = Stopwatch()..start();
GetPromptResult? result;
try {
return result = await impl(request);
} finally {
watch.stop();
try {
analytics.send(
Event.dartMCPEvent(
client: clientInfo.name,
clientVersion: clientInfo.version,
serverVersion: implementation.version,
type: AnalyticsEvent.getPrompt.name,
additionalData: GetPromptMetrics(
name: request.name,
success: result != null && result.messages.isNotEmpty,
elapsedMilliseconds: watch.elapsedMilliseconds,
withArguments: request.arguments?.isNotEmpty == true,
),
),
);
} catch (e) {
log(
LoggingLevel.warning,
'Error sending analytics event: $e',
);
}
}
},
);
}
}

/// Creates a `Sink<String>` for [logFile].
Expand Down
34 changes: 33 additions & 1 deletion pkgs/dart_mcp_server/lib/src/utils/analytics.dart
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ abstract interface class AnalyticsSupport {
Analytics? get analytics;
}

enum AnalyticsEvent { callTool, readResource }
enum AnalyticsEvent { callTool, readResource, getPrompt }

/// The metrics for a resources/read MCP handler.
final class ReadResourceMetrics extends CustomMetrics {
Expand Down Expand Up @@ -43,6 +43,36 @@ final class ReadResourceMetrics extends CustomMetrics {
};
}

/// The metrics for a prompts/get MCP handler.
final class GetPromptMetrics extends CustomMetrics {
/// The name of the prompt that was retrieved.
final String name;

/// Whether or not the prompt was given with arguments.
final bool withArguments;

/// The time it took to generate the prompt.
final int elapsedMilliseconds;

/// Whether or not the prompt call succeeded.
final bool success;

GetPromptMetrics({
required this.name,
required this.withArguments,
required this.elapsedMilliseconds,
required this.success,
});

@override
Map<String, Object> toMap() => {
_name: name,
_withArguments: withArguments,
_elapsedMilliseconds: elapsedMilliseconds,
_success: success,
};
}

/// The metrics for a tools/call MCP handler.
final class CallToolMetrics extends CustomMetrics {
/// The name of the tool that was invoked.
Expand Down Expand Up @@ -108,5 +138,7 @@ const _elapsedMilliseconds = 'elapsedMilliseconds';
const _failureReason = 'failureReason';
const _kind = 'kind';
const _length = 'length';
const _name = 'name';
const _success = 'success';
const _tool = 'tool';
const _withArguments = 'withArguments';
123 changes: 121 additions & 2 deletions pkgs/dart_mcp_server/test/dart_tooling_mcp_server_test.dart
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ void main() {
'client': server.clientInfo.name,
'clientVersion': server.clientInfo.version,
'serverVersion': server.implementation.version,
'type': 'callTool',
'type': AnalyticsEvent.callTool.name,
'tool': 'hello',
'success': true,
'elapsedMilliseconds': isA<int>(),
Expand Down Expand Up @@ -85,7 +85,7 @@ void main() {
'client': server.clientInfo.name,
'clientVersion': server.clientInfo.version,
'serverVersion': server.implementation.version,
'type': 'callTool',
'type': AnalyticsEvent.callTool.name,
'tool': tool.name,
'success': false,
'elapsedMilliseconds': isA<int>(),
Expand All @@ -96,6 +96,125 @@ void main() {
}
});

group('are sent for prompts', () {
final helloPrompt = Prompt(
name: 'hello',
arguments: [PromptArgument(name: 'name', required: false)],
);
GetPromptResult getHelloPrompt(GetPromptRequest request) {
assert(request.name == helloPrompt.name);
if (request.arguments?['throw'] == true) {
throw StateError('Oh no!');
}
return GetPromptResult(
messages: [
PromptMessage(
role: Role.user,
content: Content.text(text: 'hello'),
),
if (request.arguments?['name'] case final name?)
PromptMessage(
role: Role.user,
content: Content.text(text: ', my name is $name'),
),
],
);
}

setUp(() {
server.addPrompt(helloPrompt, getHelloPrompt);
});

test('with no arguments', () async {
final result = await testHarness.getPrompt(
GetPromptRequest(name: helloPrompt.name),
);
expect((result.messages.single.content as TextContent).text, 'hello');
expect(result.messages.single.role, Role.user);
expect(
analytics.sentEvents.single,
isA<Event>()
.having((e) => e.eventName, 'eventName', DashEvent.dartMCPEvent)
.having(
(e) => e.eventData,
'eventData',
equals({
'client': server.clientInfo.name,
'clientVersion': server.clientInfo.version,
'serverVersion': server.implementation.version,
'type': AnalyticsEvent.getPrompt.name,
'name': helloPrompt.name,
'success': true,
'elapsedMilliseconds': isA<int>(),
'withArguments': false,
}),
),
);
});

test('with arguments', () async {
final result = await testHarness.getPrompt(
GetPromptRequest(name: helloPrompt.name, arguments: {'name': 'Bob'}),
);
expect((result.messages[0].content as TextContent).text, 'hello');
expect(result.messages[0].role, Role.user);
expect(
(result.messages[1].content as TextContent).text,
', my name is Bob',
);
expect(result.messages[1].role, Role.user);
expect(
analytics.sentEvents.single,
isA<Event>()
.having((e) => e.eventName, 'eventName', DashEvent.dartMCPEvent)
.having(
(e) => e.eventData,
'eventData',
equals({
'client': server.clientInfo.name,
'clientVersion': server.clientInfo.version,
'serverVersion': server.implementation.version,
'type': AnalyticsEvent.getPrompt.name,
'name': helloPrompt.name,
'success': true,
'elapsedMilliseconds': isA<int>(),
'withArguments': true,
}),
),
);
});

test('even if they throw', () async {
try {
await testHarness.getPrompt(
GetPromptRequest(
name: helloPrompt.name,
arguments: {'throw': true},
),
);
} catch (_) {}
expect(
analytics.sentEvents.single,
isA<Event>()
.having((e) => e.eventName, 'eventName', DashEvent.dartMCPEvent)
.having(
(e) => e.eventData,
'eventData',
equals({
'client': server.clientInfo.name,
'clientVersion': server.clientInfo.version,
'serverVersion': server.implementation.version,
'type': AnalyticsEvent.getPrompt.name,
'name': helloPrompt.name,
'success': false,
'elapsedMilliseconds': isA<int>(),
'withArguments': true,
}),
),
);
});
});

test('Changelog version matches dart server version', () {
final changelogFile = File('CHANGELOG.md');
expect(
Expand Down
4 changes: 4 additions & 0 deletions pkgs/dart_mcp_server/test/test_harness.dart
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,10 @@ class TestHarness {
await Future<void>.delayed(Duration(milliseconds: 100 * tryCount));
}
}

/// Calls [getPrompt] on the [mcpServerConnection].
Future<GetPromptResult> getPrompt(GetPromptRequest request) =>
mcpServerConnection.getPrompt(request);
}

/// The debug session for a single app.
Expand Down