chore: emit strands metrics

Author: poshinchen

src/strands/event_loop/event_loop.py

@@ -13,6 +13,7 @@
 from functools import partial
 from typing import Any, Callable, Dict, List, Optional, Tuple, cast
 
+from ..telemetry import MetricsClient
 from ..telemetry.metrics import EventLoopMetrics, Trace
 from ..telemetry.tracer import get_tracer
 from ..tools.executor import run_tools, validate_and_prepare_tools
@@ -105,10 +106,14 @@ def event_loop_cycle(
     kwargs["event_loop_cycle_id"] = uuid.uuid4()
 
     event_loop_metrics: EventLoopMetrics = kwargs.get("event_loop_metrics", EventLoopMetrics())
-
+    metrics_client = MetricsClient()
     # Initialize state and get cycle trace
     kwargs = initialize_state(**kwargs)
-    cycle_start_time, cycle_trace = event_loop_metrics.start_cycle()
+
+    attributes = {"event_loop_cycle_id": str(kwargs.get("event_loop_cycle_id"))}
+    cycle_start_time, cycle_trace = event_loop_metrics.start_cycle(metrics_client)
+    metrics_client.event_loop_cycle_count.add(1, attributes=attributes)
+    metrics_client.event_loop_start_cycle.add(1, attributes=attributes)
     kwargs["event_loop_cycle_trace"] = cycle_trace
 
     callback_handler(start=True)
@@ -227,7 +232,7 @@ def event_loop_cycle(
             )
 
         # End the cycle and return results
-        event_loop_metrics.end_cycle(cycle_start_time, cycle_trace)
+        event_loop_metrics.end_cycle(cycle_start_time, cycle_trace, metrics_client)
         if cycle_span:
             tracer.end_event_loop_cycle_span(
                 span=cycle_span,
@@ -380,7 +385,7 @@ def _handle_tool_execution(
 
     if not tool_uses:
         return stop_reason, message, event_loop_metrics, kwargs["request_state"]
-
+    metrics_client = MetricsClient()
     tool_handler_process = partial(
         tool_handler.process,
         messages=messages,
@@ -418,7 +423,7 @@ def _handle_tool_execution(
         tracer.end_event_loop_cycle_span(span=cycle_span, message=message, tool_result_message=tool_result_message)
 
     if kwargs["request_state"].get("stop_event_loop", False):
-        event_loop_metrics.end_cycle(cycle_start_time, cycle_trace)
+        event_loop_metrics.end_cycle(cycle_start_time, cycle_trace, metrics_client)
         return stop_reason, message, event_loop_metrics, kwargs["request_state"]
 
     return recurse_event_loop(

src/strands/telemetry/metrics.py

@@ -7,7 +7,7 @@
 from typing import Any, Dict, Iterable, List, Optional, Set, Tuple
 
 import opentelemetry.metrics as metrics_api
-from opentelemetry.metrics import Counter, Meter
+from opentelemetry.metrics import Counter, Histogram, Meter
 
 from ..telemetry import metrics_constants as constants
 from ..types.content import Message
@@ -121,22 +121,34 @@ class ToolMetrics:
     error_count: int = 0
     total_time: float = 0.0
 
-    def add_call(self, tool: ToolUse, duration: float, success: bool) -> None:
+    def add_call(
+        self,
+        tool: ToolUse,
+        duration: float,
+        success: bool,
+        metrics_client: "MetricsClient",
+        attributes: Optional[Dict[str, Any]] = None,
+    ) -> None:
         """Record a new tool call with its outcome.
 
         Args:
             tool: The tool that was called.
             duration: How long the call took in seconds.
             success: Whether the call was successful.
+            metrics_client: The metrics client for recording the metrics.
+            attributes: attributes of the metrics.
         """
         self.tool = tool  # Update with latest tool state
         self.call_count += 1
         self.total_time += duration
-
+        metrics_client.tool_call_count.add(1, attributes=attributes)
+        metrics_client.tool_duration.record(duration, attributes=attributes)
         if success:
             self.success_count += 1
+            metrics_client.tool_success_count.add(1, attributes=attributes)
         else:
             self.error_count += 1
+            metrics_client.tool_error_count.add(1, attributes=attributes)
 
 
 @dataclass
@@ -159,32 +171,42 @@ class EventLoopMetrics:
     accumulated_usage: Usage = field(default_factory=lambda: Usage(inputTokens=0, outputTokens=0, totalTokens=0))
     accumulated_metrics: Metrics = field(default_factory=lambda: Metrics(latencyMs=0))
 
-    def start_cycle(self) -> Tuple[float, Trace]:
+    def start_cycle(self, metrics_client: "MetricsClient") -> Tuple[float, Trace]:
         """Start a new event loop cycle and create a trace for it.
 
         Returns:
             A tuple containing the start time and the cycle trace object.
         """
+        metrics_client.event_loop_cycle_count.add(1)
         self.cycle_count += 1
         start_time = time.time()
         cycle_trace = Trace(f"Cycle {self.cycle_count}", start_time=start_time)
         self.traces.append(cycle_trace)
         return start_time, cycle_trace
 
-    def end_cycle(self, start_time: float, cycle_trace: Trace) -> None:
+    def end_cycle(self, start_time: float, cycle_trace: Trace, metrics_client: "MetricsClient") -> None:
         """End the current event loop cycle and record its duration.
 
         Args:
             start_time: The timestamp when the cycle started.
             cycle_trace: The trace object for this cycle.
+            metrics_client: The metrics client for recording the metrics.
         """
+        metrics_client.event_loop_end_cycle.add(1)
         end_time = time.time()
         duration = end_time - start_time
+        metrics_client.event_loop_cycle_duration.record(duration)
         self.cycle_durations.append(duration)
         cycle_trace.end(end_time)
 
     def add_tool_usage(
-        self, tool: ToolUse, duration: float, tool_trace: Trace, success: bool, message: Message
+        self,
+        tool: ToolUse,
+        duration: float,
+        tool_trace: Trace,
+        success: bool,
+        message: Message,
+        metrics_client: "MetricsClient",
     ) -> None:
         """Record metrics for a tool invocation.
 
@@ -194,6 +216,7 @@ def add_tool_usage(
             tool_trace: The trace object for this tool call.
             success: Whether the tool call was successful.
             message: The message associated with the tool call.
+            metrics_client: The metrics client for recording the metrics.
         """
         tool_name = tool.get("name", "unknown_tool")
         tool_use_id = tool.get("toolUseId", "unknown")
@@ -207,8 +230,16 @@ def add_tool_usage(
         tool_trace.raw_name = f"{tool_name} - {tool_use_id}"
         tool_trace.add_message(message)
 
-        self.tool_metrics.setdefault(tool_name, ToolMetrics(tool)).add_call(tool, duration, success)
-
+        self.tool_metrics.setdefault(tool_name, ToolMetrics(tool)).add_call(
+            tool,
+            duration,
+            success,
+            metrics_client,
+            attributes={
+                "tool_name": tool_name,
+                "tool_use_id": tool_use_id,
+            },
+        )
         tool_trace.end()
 
     def update_usage(self, usage: Usage) -> None:
@@ -217,6 +248,7 @@ def update_usage(self, usage: Usage) -> None:
         Args:
             usage: The usage data to add to the accumulated totals.
         """
+        # metrics_client.token_usage.add(usage["totalTokens"])
         self.accumulated_usage["inputTokens"] += usage["inputTokens"]
         self.accumulated_usage["outputTokens"] += usage["outputTokens"]
         self.accumulated_usage["totalTokens"] += usage["totalTokens"]
@@ -371,6 +403,14 @@ class MetricsClient:
     _instance: Optional["MetricsClient"] = None
     meter: Meter
     strands_agent_invocation_count: Counter
+    event_loop_cycle_count: Counter
+    event_loop_start_cycle: Counter
+    event_loop_end_cycle: Counter
+    event_loop_cycle_duration: Histogram
+    tool_call_count: Counter
+    tool_success_count: Counter
+    tool_error_count: Counter
+    tool_duration: Histogram
 
     def __new__(cls) -> "MetricsClient":
         """Create or return the singleton instance of MetricsClient.
@@ -401,3 +441,21 @@ def create_instruments(self) -> None:
         self.strands_agent_invocation_count = self.meter.create_counter(
             name=constants.STRANDS_AGENT_INVOCATION_COUNT, unit="Count"
         )
+        self.event_loop_cycle_count = self.meter.create_counter(
+            name=constants.STRANDS_AGENT_EVENT_LOOP_CYCLE_COUNT, unit="Count"
+        )
+        self.event_loop_start_cycle = self.meter.create_counter(
+            name=constants.STRANDS_AGENT_EVENT_LOOP_START_CYCLE, unit="Count"
+        )
+        self.event_loop_end_cycle = self.meter.create_counter(
+            name=constants.STRANDS_AGENT_EVENT_LOOP_END_CYCLE, unit="Count"
+        )
+        self.event_loop_cycle_duration = self.meter.create_histogram(
+            name=constants.STRANDS_AGENT_EVENT_LOOP_CYCLE_DURATION, unit="s"
+        )
+        self.tool_call_count = self.meter.create_counter(name=constants.STRANDS_AGENT_TOOL_CALL_COUNT, unit="Count")
+        self.tool_success_count = self.meter.create_counter(
+            name=constants.STRANDS_AGENT_TOOL_SUCCESS_COUNT, unit="Count"
+        )
+        self.tool_error_count = self.meter.create_counter(name=constants.STRANDS_AGENT_TOOL_ERROR_COUNT, unit="Count")
+        self.tool_duration = self.meter.create_histogram(name=constants.STRANDS_AGENT_TOOL_DURATION, unit="s")

src/strands/telemetry/metrics_constants.py

@@ -1,3 +1,11 @@
 """Metrics that are emitted in Strands-Agent."""
 
 STRANDS_AGENT_INVOCATION_COUNT = "strands.agent.invocation_count"
+STRANDS_AGENT_EVENT_LOOP_CYCLE_COUNT = "strands.agent.event_loop.cycle_count"
+STRANDS_AGENT_EVENT_LOOP_START_CYCLE = "strands.agent.event_loop.start_cycle"
+STRANDS_AGENT_EVENT_LOOP_END_CYCLE = "strands.agent.event_loop.end_cycle"
+STRANDS_AGENT_EVENT_LOOP_CYCLE_DURATION = "strands.agent.event_loop.cycle_duration"
+STRANDS_AGENT_TOOL_CALL_COUNT = "strands.agent.tool.call_count"
+STRANDS_AGENT_TOOL_SUCCESS_COUNT = "strands.agent.tool.success_count"
+STRANDS_AGENT_TOOL_ERROR_COUNT = "strands.agent.tool.error_count"
+STRANDS_AGENT_TOOL_DURATION = "strands.agent.tool.duration"

src/strands/tools/executor.py

@@ -7,6 +7,7 @@
 
 from opentelemetry import trace
 
+from ..telemetry import MetricsClient
 from ..telemetry.metrics import EventLoopMetrics, Trace
 from ..telemetry.tracer import get_tracer
 from ..tools.tools import InvalidToolUseNameException, validate_tool_use
@@ -51,6 +52,7 @@ def _handle_tool_execution(tool: ToolUse) -> Tuple[bool, Optional[ToolResult]]:
 
         tracer = get_tracer()
         tool_call_span = tracer.start_tool_call_span(tool, parent_span)
+        metrics_client = MetricsClient()
 
         try:
             if "toolUseId" not in tool or tool["toolUseId"] not in invalid_tool_use_ids:
@@ -64,7 +66,9 @@ def _handle_tool_execution(tool: ToolUse) -> Tuple[bool, Optional[ToolResult]]:
 
                 tool_duration = time.time() - tool_start_time
                 message = Message(role="user", content=[{"toolResult": result}])
-                event_loop_metrics.add_tool_usage(tool, tool_duration, tool_trace, tool_success, message)
+                event_loop_metrics.add_tool_usage(
+                    tool, tool_duration, tool_trace, tool_success, message, metrics_client
+                )
                 cycle_trace.add_child(tool_trace)
 
             if tool_call_span:

tests/strands/telemetry/test_metrics.py

@@ -124,12 +124,19 @@ def test_trace_end(mock_time, end_time, trace):
 @pytest.fixture
 def mock_get_meter_provider():
     with mock.patch("strands.telemetry.metrics.metrics_api.get_meter_provider") as mock_get_meter_provider:
+        MetricsClient._instance = None
         meter_provider_mock = mock.MagicMock(spec=MeterProvider)
-        mock_get_meter_provider.return_value = meter_provider_mock
 
         mock_meter = mock.MagicMock()
+        mock_create_counter = mock.MagicMock()
+        mock_meter.create_counter.return_value = mock_create_counter
+
+        mock_create_histogram = mock.MagicMock()
+        mock_meter.create_histogram.return_value = mock_create_histogram
         meter_provider_mock.get_meter.return_value = mock_meter
 
+        mock_get_meter_provider.return_value = meter_provider_mock
+
         yield mock_get_meter_provider
 
 
@@ -190,11 +197,14 @@ def test_trace_to_dict(trace):
 
 
 @pytest.mark.parametrize("success", [True, False])
-def test_tool_metrics_add_call(success, tool, tool_metrics):
+def test_tool_metrics_add_call(success, tool, tool_metrics, mock_get_meter_provider):
     tool = dict(tool, **{"name": "updated"})
     duration = 1
 
-    tool_metrics.add_call(tool, duration, success)
+    metrics_client = MetricsClient()
+    attributes = {"foo": "bar"}
+
+    tool_metrics.add_call(tool, duration, success, metrics_client, attributes=attributes)
 
     tru_attrs = dataclasses.asdict(tool_metrics)
     exp_attrs = {
@@ -205,21 +215,30 @@ def test_tool_metrics_add_call(success, tool, tool_metrics):
         "total_time": duration,
     }
 
+    mock_get_meter_provider.return_value.get_meter.assert_called()
+    metrics_client.tool_call_count.add.assert_called_with(1, attributes=attributes)
+    metrics_client.tool_duration.record.assert_called_with(duration, attributes=attributes)
+    if success:
+        metrics_client.tool_success_count.add.assert_called_with(1, attributes=attributes)
     assert tru_attrs == exp_attrs
 
 
 @unittest.mock.patch.object(strands.telemetry.metrics.time, "time")
 @unittest.mock.patch.object(strands.telemetry.metrics.uuid, "uuid4")
-def test_event_loop_metrics_start_cycle(mock_uuid4, mock_time, event_loop_metrics):
+def test_event_loop_metrics_start_cycle(mock_uuid4, mock_time, event_loop_metrics, mock_get_meter_provider):
     mock_time.return_value = 1
     mock_uuid4.return_value = "i1"
 
-    tru_start_time, tru_cycle_trace = event_loop_metrics.start_cycle()
+    metrics_client = MetricsClient()
+
+    tru_start_time, tru_cycle_trace = event_loop_metrics.start_cycle(metrics_client)
     exp_start_time, exp_cycle_trace = 1, strands.telemetry.metrics.Trace("Cycle 1")
 
     tru_attrs = {"cycle_count": event_loop_metrics.cycle_count, "traces": event_loop_metrics.traces}
     exp_attrs = {"cycle_count": 1, "traces": [tru_cycle_trace]}
 
+    mock_get_meter_provider.return_value.get_meter.assert_called()
+    metrics_client.event_loop_cycle_count.add.assert_called()
     assert (
         tru_start_time == exp_start_time
         and tru_cycle_trace.to_dict() == exp_cycle_trace.to_dict()
@@ -228,10 +247,11 @@ def test_event_loop_metrics_start_cycle(mock_uuid4, mock_time, event_loop_metric
 
 
 @unittest.mock.patch.object(strands.telemetry.metrics.time, "time")
-def test_event_loop_metrics_end_cycle(mock_time, trace, event_loop_metrics):
+def test_event_loop_metrics_end_cycle(mock_time, trace, event_loop_metrics, mock_get_meter_provider):
     mock_time.return_value = 1
+    metrics_client = MetricsClient()
 
-    event_loop_metrics.end_cycle(start_time=0, cycle_trace=trace)
+    event_loop_metrics.end_cycle(start_time=0, cycle_trace=trace, metrics_client=metrics_client)
 
     tru_cycle_durations = event_loop_metrics.cycle_durations
     exp_cycle_durations = [1]
@@ -243,16 +263,22 @@ def test_event_loop_metrics_end_cycle(mock_time, trace, event_loop_metrics):
 
     assert tru_trace_end_time == exp_trace_end_time
 
+    mock_get_meter_provider.return_value.get_meter.assert_called()
+    metrics_client.event_loop_end_cycle.add.assert_called_with(1)
+    metrics_client.event_loop_cycle_duration.record.assert_called()
+
 
 @unittest.mock.patch.object(strands.telemetry.metrics.time, "time")
-def test_event_loop_metrics_add_tool_usage(mock_time, trace, tool, event_loop_metrics):
+def test_event_loop_metrics_add_tool_usage(mock_time, trace, tool, event_loop_metrics, mock_get_meter_provider):
+    metrics_client = MetricsClient()
     mock_time.return_value = 1
-
     duration = 1
     success = True
     message = {"role": "user", "content": [{"toolResult": {"toolUseId": "123", "tool_name": "tool1"}}]}
 
-    event_loop_metrics.add_tool_usage(tool, duration, trace, success, message)
+    event_loop_metrics.add_tool_usage(tool, duration, trace, success, message, metrics_client)
+
+    mock_get_meter_provider.return_value.get_meter.assert_called()
 
     tru_event_loop_metrics_attrs = {"tool_metrics": event_loop_metrics.tool_metrics}
     exp_event_loop_metrics_attrs = {
@@ -312,12 +338,13 @@ def test_event_loop_metrics_update_metrics(metrics, event_loop_metrics):
     assert tru_metrics == exp_metrics
 
 
-def test_event_loop_metrics_get_summary(trace, tool, event_loop_metrics):
+def test_event_loop_metrics_get_summary(trace, tool, event_loop_metrics, mock_get_meter_provider):
     duration = 1
     success = True
     message = {"role": "user", "content": [{"toolResult": {"toolUseId": "123", "tool_name": "tool1"}}]}
+    metrics_client = MetricsClient()
 
-    event_loop_metrics.add_tool_usage(tool, duration, trace, success, message)
+    event_loop_metrics.add_tool_usage(tool, duration, trace, success, message, metrics_client)
 
     tru_summary = event_loop_metrics.get_summary()
     exp_summary = {