chore: emit strands metrics

Author: poshinchen

src/strands/event_loop/event_loop.py

@@ -13,6 +13,7 @@
 from functools import partial
 from typing import Any, Callable, Dict, List, Optional, Tuple, cast
 
+from ..telemetry import MetricsClient
 from ..telemetry.metrics import EventLoopMetrics, Trace
 from ..telemetry.tracer import get_tracer
 from ..tools.executor import run_tools, validate_and_prepare_tools
@@ -105,10 +106,14 @@ def event_loop_cycle(
     kwargs["event_loop_cycle_id"] = uuid.uuid4()
 
     event_loop_metrics: EventLoopMetrics = kwargs.get("event_loop_metrics", EventLoopMetrics())
-
+    metrics_client = MetricsClient()
     # Initialize state and get cycle trace
     kwargs = initialize_state(**kwargs)
-    cycle_start_time, cycle_trace = event_loop_metrics.start_cycle()
+
+    attributes = {"event_loop_cycle_id": str(kwargs.get("event_loop_cycle_id"))}
+    cycle_start_time, cycle_trace = event_loop_metrics.start_cycle(metrics_client)
+    metrics_client.event_loop_cycle_count.add(1, attributes=attributes)
+    metrics_client.event_loop_start_cycle.add(1, attributes=attributes)
     kwargs["event_loop_cycle_trace"] = cycle_trace
 
     callback_handler(start=True)
@@ -191,8 +196,8 @@ def event_loop_cycle(
         callback_handler(message=message)
 
         # Update metrics
-        event_loop_metrics.update_usage(usage)
-        event_loop_metrics.update_metrics(metrics)
+        event_loop_metrics.update_usage(usage, metrics_client)
+        event_loop_metrics.update_metrics(metrics, metrics_client)
 
         # If the model is requesting to use tools
         if stop_reason == "tool_use":
@@ -227,7 +232,7 @@ def event_loop_cycle(
             )
 
         # End the cycle and return results
-        event_loop_metrics.end_cycle(cycle_start_time, cycle_trace)
+        event_loop_metrics.end_cycle(cycle_start_time, cycle_trace, metrics_client)
         if cycle_span:
             tracer.end_event_loop_cycle_span(
                 span=cycle_span,
@@ -380,7 +385,7 @@ def _handle_tool_execution(
 
     if not tool_uses:
         return stop_reason, message, event_loop_metrics, kwargs["request_state"]
-
+    metrics_client = MetricsClient()
     tool_handler_process = partial(
         tool_handler.process,
         messages=messages,
@@ -418,7 +423,7 @@ def _handle_tool_execution(
         tracer.end_event_loop_cycle_span(span=cycle_span, message=message, tool_result_message=tool_result_message)
 
     if kwargs["request_state"].get("stop_event_loop", False):
-        event_loop_metrics.end_cycle(cycle_start_time, cycle_trace)
+        event_loop_metrics.end_cycle(cycle_start_time, cycle_trace, metrics_client)
         return stop_reason, message, event_loop_metrics, kwargs["request_state"]
 
     return recurse_event_loop(

src/strands/telemetry/metrics.py

@@ -7,7 +7,7 @@
 from typing import Any, Dict, Iterable, List, Optional, Set, Tuple
 
 import opentelemetry.metrics as metrics_api
-from opentelemetry.metrics import Counter, Meter
+from opentelemetry.metrics import Counter, Histogram, Meter
 
 from ..telemetry import metrics_constants as constants
 from ..types.content import Message
@@ -121,22 +121,34 @@ class ToolMetrics:
     error_count: int = 0
     total_time: float = 0.0
 
-    def add_call(self, tool: ToolUse, duration: float, success: bool) -> None:
+    def add_call(
+        self,
+        tool: ToolUse,
+        duration: float,
+        success: bool,
+        metrics_client: "MetricsClient",
+        attributes: Optional[Dict[str, Any]] = None,
+    ) -> None:
         """Record a new tool call with its outcome.
 
         Args:
             tool: The tool that was called.
             duration: How long the call took in seconds.
             success: Whether the call was successful.
+            metrics_client: The metrics client for recording the metrics.
+            attributes: attributes of the metrics.
         """
         self.tool = tool  # Update with latest tool state
         self.call_count += 1
         self.total_time += duration
-
+        metrics_client.tool_call_count.add(1, attributes=attributes)
+        metrics_client.tool_duration.record(duration, attributes=attributes)
         if success:
             self.success_count += 1
+            metrics_client.tool_success_count.add(1, attributes=attributes)
         else:
             self.error_count += 1
+            metrics_client.tool_error_count.add(1, attributes=attributes)
 
 
 @dataclass
@@ -159,32 +171,42 @@ class EventLoopMetrics:
     accumulated_usage: Usage = field(default_factory=lambda: Usage(inputTokens=0, outputTokens=0, totalTokens=0))
     accumulated_metrics: Metrics = field(default_factory=lambda: Metrics(latencyMs=0))
 
-    def start_cycle(self) -> Tuple[float, Trace]:
+    def start_cycle(self, metrics_client: "MetricsClient") -> Tuple[float, Trace]:
         """Start a new event loop cycle and create a trace for it.
 
         Returns:
             A tuple containing the start time and the cycle trace object.
         """
+        metrics_client.event_loop_cycle_count.add(1)
         self.cycle_count += 1
         start_time = time.time()
         cycle_trace = Trace(f"Cycle {self.cycle_count}", start_time=start_time)
         self.traces.append(cycle_trace)
         return start_time, cycle_trace
 
-    def end_cycle(self, start_time: float, cycle_trace: Trace) -> None:
+    def end_cycle(self, start_time: float, cycle_trace: Trace, metrics_client: "MetricsClient") -> None:
         """End the current event loop cycle and record its duration.
 
         Args:
             start_time: The timestamp when the cycle started.
             cycle_trace: The trace object for this cycle.
+            metrics_client: The metrics client for recording the metrics.
         """
+        metrics_client.event_loop_end_cycle.add(1)
         end_time = time.time()
         duration = end_time - start_time
+        metrics_client.event_loop_cycle_duration.record(duration)
         self.cycle_durations.append(duration)
         cycle_trace.end(end_time)
 
     def add_tool_usage(
-        self, tool: ToolUse, duration: float, tool_trace: Trace, success: bool, message: Message
+        self,
+        tool: ToolUse,
+        duration: float,
+        tool_trace: Trace,
+        success: bool,
+        message: Message,
+        metrics_client: "MetricsClient",
     ) -> None:
         """Record metrics for a tool invocation.
 
@@ -194,6 +216,7 @@ def add_tool_usage(
             tool_trace: The trace object for this tool call.
             success: Whether the tool call was successful.
             message: The message associated with the tool call.
+            metrics_client: The metrics client for recording the metrics.
         """
         tool_name = tool.get("name", "unknown_tool")
         tool_use_id = tool.get("toolUseId", "unknown")
@@ -207,26 +230,39 @@ def add_tool_usage(
         tool_trace.raw_name = f"{tool_name} - {tool_use_id}"
         tool_trace.add_message(message)
 
-        self.tool_metrics.setdefault(tool_name, ToolMetrics(tool)).add_call(tool, duration, success)
-
+        self.tool_metrics.setdefault(tool_name, ToolMetrics(tool)).add_call(
+            tool,
+            duration,
+            success,
+            metrics_client,
+            attributes={
+                "tool_name": tool_name,
+                "tool_use_id": tool_use_id,
+            },
+        )
         tool_trace.end()
 
-    def update_usage(self, usage: Usage) -> None:
+    def update_usage(self, usage: Usage, metrics_client: "MetricsClient") -> None:
         """Update the accumulated token usage with new usage data.
 
         Args:
             usage: The usage data to add to the accumulated totals.
+            metrics_client: The metrics client for recording the metrics.
         """
+        metrics_client.event_loop_input_tokens.record(usage["inputTokens"])
+        metrics_client.event_loop_output_tokens.record(usage["outputTokens"])
         self.accumulated_usage["inputTokens"] += usage["inputTokens"]
         self.accumulated_usage["outputTokens"] += usage["outputTokens"]
         self.accumulated_usage["totalTokens"] += usage["totalTokens"]
 
-    def update_metrics(self, metrics: Metrics) -> None:
+    def update_metrics(self, metrics: Metrics, metrics_client: "MetricsClient") -> None:
         """Update the accumulated performance metrics with new metrics data.
 
         Args:
             metrics: The metrics data to add to the accumulated totals.
+            metrics_client: The metrics client for recording the metrics.
         """
+        metrics_client.event_loop_latency.record(metrics["latencyMs"])
         self.accumulated_metrics["latencyMs"] += metrics["latencyMs"]
 
     def get_summary(self) -> Dict[str, Any]:
@@ -370,7 +406,18 @@ class MetricsClient:
 
     _instance: Optional["MetricsClient"] = None
     meter: Meter
-    strands_agent_invocation_count: Counter
+    event_loop_cycle_count: Counter
+    event_loop_start_cycle: Counter
+    event_loop_end_cycle: Counter
+    event_loop_cycle_duration: Histogram
+    event_loop_latency: Histogram
+    event_loop_input_tokens: Histogram
+    event_loop_output_tokens: Histogram
+
+    tool_call_count: Counter
+    tool_success_count: Counter
+    tool_error_count: Counter
+    tool_duration: Histogram
 
     def __new__(cls) -> "MetricsClient":
         """Create or return the singleton instance of MetricsClient.
@@ -398,6 +445,24 @@ def __init__(self) -> None:
 
     def create_instruments(self) -> None:
         """Create and initialize all OpenTelemetry metric instruments."""
-        self.strands_agent_invocation_count = self.meter.create_counter(
-            name=constants.STRANDS_AGENT_INVOCATION_COUNT, unit="Count"
+        self.event_loop_cycle_count = self.meter.create_counter(
+            name=constants.STRANDS_EVENT_LOOP_CYCLE_COUNT, unit="Count"
+        )
+        self.event_loop_start_cycle = self.meter.create_counter(
+            name=constants.STRANDS_EVENT_LOOP_START_CYCLE, unit="Count"
+        )
+        self.event_loop_end_cycle = self.meter.create_counter(name=constants.STRANDS_EVENT_LOOP_END_CYCLE, unit="Count")
+        self.event_loop_cycle_duration = self.meter.create_histogram(
+            name=constants.STRANDS_EVENT_LOOP_CYCLE_DURATION, unit="s"
+        )
+        self.event_loop_latency = self.meter.create_histogram(name=constants.STRANDS_EVENT_LOOP_LATENCY, unit="ms")
+        self.tool_call_count = self.meter.create_counter(name=constants.STRANDS_TOOL_CALL_COUNT, unit="Count")
+        self.tool_success_count = self.meter.create_counter(name=constants.STRANDS_TOOL_SUCCESS_COUNT, unit="Count")
+        self.tool_error_count = self.meter.create_counter(name=constants.STRANDS_TOOL_ERROR_COUNT, unit="Count")
+        self.tool_duration = self.meter.create_histogram(name=constants.STRANDS_TOOL_DURATION, unit="s")
+        self.event_loop_input_tokens = self.meter.create_histogram(
+            name=constants.STRANDS_EVENT_LOOP_INPUT_TOKENS, unit="token"
+        )
+        self.event_loop_output_tokens = self.meter.create_histogram(
+            name=constants.STRANDS_EVENT_LOOP_OUTPUT_TOKENS, unit="token"
         )

src/strands/telemetry/metrics_constants.py

@@ -1,3 +1,15 @@
-"""Metrics that are emitted in Strands-Agent."""
+"""Metrics that are emitted in Strands-Agents."""
 
-STRANDS_AGENT_INVOCATION_COUNT = "strands.agent.invocation_count"
+STRANDS_EVENT_LOOP_CYCLE_COUNT = "strands.event_loop.cycle_count"
+STRANDS_EVENT_LOOP_START_CYCLE = "strands.event_loop.start_cycle"
+STRANDS_EVENT_LOOP_END_CYCLE = "strands.event_loop.end_cycle"
+STRANDS_TOOL_CALL_COUNT = "strands.tool.call_count"
+STRANDS_TOOL_SUCCESS_COUNT = "strands.tool.success_count"
+STRANDS_TOOL_ERROR_COUNT = "strands.tool.error_count"
+
+# Histograms
+STRANDS_EVENT_LOOP_LATENCY = "strands.event_loop.latency"
+STRANDS_TOOL_DURATION = "strands.tool.duration"
+STRANDS_EVENT_LOOP_CYCLE_DURATION = "strands.event_loop.cycle_duration"
+STRANDS_EVENT_LOOP_INPUT_TOKENS = "strands.event_loop.input.tokens"
+STRANDS_EVENT_LOOP_OUTPUT_TOKENS = "strands.event_loop.output.tokens"

src/strands/tools/executor.py

@@ -7,6 +7,7 @@
 
 from opentelemetry import trace
 
+from ..telemetry import MetricsClient
 from ..telemetry.metrics import EventLoopMetrics, Trace
 from ..telemetry.tracer import get_tracer
 from ..tools.tools import InvalidToolUseNameException, validate_tool_use
@@ -51,6 +52,7 @@ def _handle_tool_execution(tool: ToolUse) -> Tuple[bool, Optional[ToolResult]]:
 
         tracer = get_tracer()
         tool_call_span = tracer.start_tool_call_span(tool, parent_span)
+        metrics_client = MetricsClient()
 
         try:
             if "toolUseId" not in tool or tool["toolUseId"] not in invalid_tool_use_ids:
@@ -64,7 +66,9 @@ def _handle_tool_execution(tool: ToolUse) -> Tuple[bool, Optional[ToolResult]]:
 
                 tool_duration = time.time() - tool_start_time
                 message = Message(role="user", content=[{"toolResult": result}])
-                event_loop_metrics.add_tool_usage(tool, tool_duration, tool_trace, tool_success, message)
+                event_loop_metrics.add_tool_usage(
+                    tool, tool_duration, tool_trace, tool_success, message, metrics_client
+                )
                 cycle_trace.add_child(tool_trace)
 
             if tool_call_span: