fix(llmobs): fix parsing stream when response.completed chunk is missing (#13850)

lievan · happynancee · commit d27975d1795d · 2025-07-07T10:55:23.000-04:00
The openai responses api stream returns any of the following chunk types that have `response` field. - `response.completed`, `response.incomplete`, `response.failed`, `response.created`, `response.in_progress` https://platform.openai.com/docs/api-reference/responses-streaming/response The `response` field is updated when each chunk arrives, so we always want to grab the _latest_ one to read data for setting attributes on our llm obs span. However, currently, we are only accounting for `response.completed`. This actually results in an `IndexError` when the `response.completed` chunk is not present. This pr updates the logic always use the latest `response` field of the streamed chunks so we support all the chunk types. ### Note We bump the test agent version so that it supports our new vcr proxy logic introduced [here](DataDog/dd-apm-test-agent@9349315) ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)
diff --git a/ddtrace/contrib/internal/openai/utils.py b/ddtrace/contrib/internal/openai/utils.py
@@ -277,8 +277,8 @@ def _loop_handler(span, chunk, streamed_chunks):
         span.set_tag_str("openai.response.model", model)
 
     response = getattr(chunk, "response", None)
-    if getattr(chunk, "type", "") == "response.completed":
-        streamed_chunks[0].append(response)
+    if response is not None:
+        streamed_chunks[0].insert(0, response)
 
     # Completions/chat completions are returned as `choices`
     for choice in getattr(chunk, "choices", []):
@@ -292,7 +292,7 @@ def _process_finished_stream(integration, span, kwargs, streamed_chunks, operati
     request_messages = kwargs.get("messages", None)
     try:
         if operation_type == "response":
-            formatted_completions = streamed_chunks[0][0]
+            formatted_completions = streamed_chunks[0][0] if streamed_chunks and streamed_chunks[0] else None
         elif operation_type == "completion":
             formatted_completions = [
                 openai_construct_completion_from_streamed_chunks(choice) for choice in streamed_chunks
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -228,4 +228,4 @@ services:
             - "127.0.0.1:7005:7005"
 
 volumes:
-    ddagent:
+    ddagent:
diff --git a/releasenotes/notes/fix-responses-stream-84dfe439b0d9f635.yaml b/releasenotes/notes/fix-responses-stream-84dfe439b0d9f635.yaml
@@ -0,0 +1,4 @@
+---
+fixes:
+  - |
+    LLM Observability: This fix resolves an issue where incomplete streamed responses returned from OpenAI responses API caused an index error with LLM Observability tracing.
diff --git a/tests/contrib/openai/cassettes/v1/response_stream_incomplete.yaml b/tests/contrib/openai/cassettes/v1/response_stream_incomplete.yaml
@@ -0,0 +1,217 @@
+interactions:
+- request:
+    body: '{"input":"Give me a multi paragraph narrative on the life of a car","max_output_tokens":16,"model":"gpt-4o","stream":true,"temperature":0.1}'
+    headers:
+      accept:
+      - application/json
+      accept-encoding:
+      - gzip, deflate
+      connection:
+      - keep-alive
+      content-length:
+      - '140'
+      content-type:
+      - application/json
+      host:
+      - api.openai.com
+      user-agent:
+      - OpenAI/Python 1.91.0
+      x-stainless-arch:
+      - arm64
+      x-stainless-async:
+      - 'false'
+      x-stainless-lang:
+      - python
+      x-stainless-os:
+      - MacOS
+      x-stainless-package-version:
+      - 1.91.0
+      x-stainless-read-timeout:
+      - '600'
+      x-stainless-retry-count:
+      - '0'
+      x-stainless-runtime:
+      - CPython
+      x-stainless-runtime-version:
+      - 3.10.13
+    method: POST
+    uri: https://api.openai.com/v1/responses
+  response:
+    body:
+      string: 'event: response.created
+
+        data: {"type":"response.created","sequence_number":0,"response":{"id":"resp_6866d2dec420819c92534dcf75e475120847a84b87aad89c","object":"response","created_at":1751569118,"status":"in_progress","background":false,"error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":16,"max_tool_calls":null,"model":"gpt-4o-2024-08-06","output":[],"parallel_tool_calls":true,"previous_response_id":null,"reasoning":{"effort":null,"summary":null},"service_tier":"auto","store":false,"temperature":0.1,"text":{"format":{"type":"text"}},"tool_choice":"auto","tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}}}
+
+
+        event: response.in_progress
+
+        data: {"type":"response.in_progress","sequence_number":1,"response":{"id":"resp_6866d2dec420819c92534dcf75e475120847a84b87aad89c","object":"response","created_at":1751569118,"status":"in_progress","background":false,"error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":16,"max_tool_calls":null,"model":"gpt-4o-2024-08-06","output":[],"parallel_tool_calls":true,"previous_response_id":null,"reasoning":{"effort":null,"summary":null},"service_tier":"auto","store":false,"temperature":0.1,"text":{"format":{"type":"text"}},"tool_choice":"auto","tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}}}
+
+
+        event: response.output_item.added
+
+        data: {"type":"response.output_item.added","sequence_number":2,"output_index":0,"item":{"id":"msg_6866d2df1c0c819c885ca369ba853e7f0847a84b87aad89c","type":"message","status":"in_progress","content":[],"role":"assistant"}}
+
+
+        event: response.content_part.added
+
+        data: {"type":"response.content_part.added","sequence_number":3,"item_id":"msg_6866d2df1c0c819c885ca369ba853e7f0847a84b87aad89c","output_index":0,"content_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":""}}
+
+
+        event: response.output_text.delta
+
+        data: {"type":"response.output_text.delta","sequence_number":4,"item_id":"msg_6866d2df1c0c819c885ca369ba853e7f0847a84b87aad89c","output_index":0,"content_index":0,"delta":"In","logprobs":[]}
+
+
+        event: response.output_text.delta
+
+        data: {"type":"response.output_text.delta","sequence_number":5,"item_id":"msg_6866d2df1c0c819c885ca369ba853e7f0847a84b87aad89c","output_index":0,"content_index":0,"delta":"
+        the","logprobs":[]}
+
+
+        event: response.output_text.delta
+
+        data: {"type":"response.output_text.delta","sequence_number":6,"item_id":"msg_6866d2df1c0c819c885ca369ba853e7f0847a84b87aad89c","output_index":0,"content_index":0,"delta":"
+        bustling","logprobs":[]}
+
+
+        event: response.output_text.delta
+
+        data: {"type":"response.output_text.delta","sequence_number":7,"item_id":"msg_6866d2df1c0c819c885ca369ba853e7f0847a84b87aad89c","output_index":0,"content_index":0,"delta":"
+        city","logprobs":[]}
+
+
+        event: response.output_text.delta
+
+        data: {"type":"response.output_text.delta","sequence_number":8,"item_id":"msg_6866d2df1c0c819c885ca369ba853e7f0847a84b87aad89c","output_index":0,"content_index":0,"delta":"
+        of","logprobs":[]}
+
+
+        event: response.output_text.delta
+
+        data: {"type":"response.output_text.delta","sequence_number":9,"item_id":"msg_6866d2df1c0c819c885ca369ba853e7f0847a84b87aad89c","output_index":0,"content_index":0,"delta":"
+        Detroit","logprobs":[]}
+
+
+        event: response.output_text.delta
+
+        data: {"type":"response.output_text.delta","sequence_number":10,"item_id":"msg_6866d2df1c0c819c885ca369ba853e7f0847a84b87aad89c","output_index":0,"content_index":0,"delta":",","logprobs":[]}
+
+
+        event: response.output_text.delta
+
+        data: {"type":"response.output_text.delta","sequence_number":11,"item_id":"msg_6866d2df1c0c819c885ca369ba853e7f0847a84b87aad89c","output_index":0,"content_index":0,"delta":"
+        a","logprobs":[]}
+
+
+        event: response.output_text.delta
+
+        data: {"type":"response.output_text.delta","sequence_number":12,"item_id":"msg_6866d2df1c0c819c885ca369ba853e7f0847a84b87aad89c","output_index":0,"content_index":0,"delta":"
+        sleek","logprobs":[]}
+
+
+        event: response.output_text.delta
+
+        data: {"type":"response.output_text.delta","sequence_number":13,"item_id":"msg_6866d2df1c0c819c885ca369ba853e7f0847a84b87aad89c","output_index":0,"content_index":0,"delta":",","logprobs":[]}
+
+
+        event: response.output_text.delta
+
+        data: {"type":"response.output_text.delta","sequence_number":14,"item_id":"msg_6866d2df1c0c819c885ca369ba853e7f0847a84b87aad89c","output_index":0,"content_index":0,"delta":"
+        metallic","logprobs":[]}
+
+
+        event: response.output_text.delta
+
+        data: {"type":"response.output_text.delta","sequence_number":15,"item_id":"msg_6866d2df1c0c819c885ca369ba853e7f0847a84b87aad89c","output_index":0,"content_index":0,"delta":"
+        blue","logprobs":[]}
+
+
+        event: response.output_text.delta
+
+        data: {"type":"response.output_text.delta","sequence_number":16,"item_id":"msg_6866d2df1c0c819c885ca369ba853e7f0847a84b87aad89c","output_index":0,"content_index":0,"delta":"
+        sedan","logprobs":[]}
+
+
+        event: response.output_text.delta
+
+        data: {"type":"response.output_text.delta","sequence_number":17,"item_id":"msg_6866d2df1c0c819c885ca369ba853e7f0847a84b87aad89c","output_index":0,"content_index":0,"delta":"
+        rolled","logprobs":[]}
+
+
+        event: response.output_text.delta
+
+        data: {"type":"response.output_text.delta","sequence_number":18,"item_id":"msg_6866d2df1c0c819c885ca369ba853e7f0847a84b87aad89c","output_index":0,"content_index":0,"delta":"
+        off","logprobs":[]}
+
+
+        event: response.output_text.delta
+
+        data: {"type":"response.output_text.delta","sequence_number":19,"item_id":"msg_6866d2df1c0c819c885ca369ba853e7f0847a84b87aad89c","output_index":0,"content_index":0,"delta":"
+        the","logprobs":[]}
+
+
+        event: response.output_text.done
+
+        data: {"type":"response.output_text.done","sequence_number":20,"item_id":"msg_6866d2df1c0c819c885ca369ba853e7f0847a84b87aad89c","output_index":0,"content_index":0,"text":"In
+        the bustling city of Detroit, a sleek, metallic blue sedan rolled off the","logprobs":[]}
+
+
+        event: response.content_part.done
+
+        data: {"type":"response.content_part.done","sequence_number":21,"item_id":"msg_6866d2df1c0c819c885ca369ba853e7f0847a84b87aad89c","output_index":0,"content_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":"In
+        the bustling city of Detroit, a sleek, metallic blue sedan rolled off the"}}
+
+
+        event: response.output_item.done
+
+        data: {"type":"response.output_item.done","sequence_number":22,"output_index":0,"item":{"id":"msg_6866d2df1c0c819c885ca369ba853e7f0847a84b87aad89c","type":"message","status":"incomplete","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"In
+        the bustling city of Detroit, a sleek, metallic blue sedan rolled off the"}],"role":"assistant"}}
+
+
+        event: response.incomplete
+
+        data: {"type":"response.incomplete","sequence_number":23,"response":{"id":"resp_6866d2dec420819c92534dcf75e475120847a84b87aad89c","object":"response","created_at":1751569118,"status":"incomplete","background":false,"error":null,"incomplete_details":{"reason":"max_output_tokens"},"instructions":null,"max_output_tokens":16,"max_tool_calls":null,"model":"gpt-4o-2024-08-06","output":[{"id":"msg_6866d2df1c0c819c885ca369ba853e7f0847a84b87aad89c","type":"message","status":"incomplete","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"In
+        the bustling city of Detroit, a sleek, metallic blue sedan rolled off the"}],"role":"assistant"}],"parallel_tool_calls":true,"previous_response_id":null,"reasoning":{"effort":null,"summary":null},"service_tier":"default","store":false,"temperature":0.1,"text":{"format":{"type":"text"}},"tool_choice":"auto","tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":0,"input_tokens_details":{"cached_tokens":0},"output_tokens":0,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":0},"user":null,"metadata":{}}}
+
+
+        '
+    headers:
+      CF-RAY:
+      - 95989d900ebc15c7-EWR
+      Connection:
+      - keep-alive
+      Content-Type:
+      - text/event-stream; charset=utf-8
+      Date:
+      - Thu, 03 Jul 2025 18:58:38 GMT
+      Server:
+      - cloudflare
+      Set-Cookie:
+      - __cf_bm=KpYG1oRKkQLyyjVvJMwoM7ql79VaamURrWrTFo.aTE0-1751569118-1.0.1.1-KGbIEiERemwjKrv8ycZmhv0cRJOMMKhFVJv.o0u1rPJqtnugj.3FF_6iz78OARuG5mZZ0ohKG5geXbCQ3uC9eZ0wEn0sBFq3X2qgV5BUl44;
+        path=/; expires=Thu, 03-Jul-25 19:28:38 GMT; domain=.api.openai.com; HttpOnly;
+        Secure; SameSite=None
+      - _cfuvid=r8PrX5IPe5I9EgCb9rpkzUITgLbT65.Q573NUBXW_iY-1751569118795-0.0.1.1-604800000;
+        path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None
+      Transfer-Encoding:
+      - chunked
+      X-Content-Type-Options:
+      - nosniff
+      alt-svc:
+      - h3=":443"; ma=86400
+      cf-cache-status:
+      - DYNAMIC
+      openai-organization:
+      - datadog-staging
+      openai-processing-ms:
+      - '27'
+      openai-version:
+      - '2020-10-01'
+      strict-transport-security:
+      - max-age=31536000; includeSubDomains; preload
+      x-request-id:
+      - req_e5f20714325d29dc7b7e40b8a587ff3d
+    status:
+      code: 200
+      message: OK
+version: 1
diff --git a/tests/contrib/openai/test_openai_llmobs.py b/tests/contrib/openai/test_openai_llmobs.py
@@ -1120,6 +1120,57 @@ def test_response_stream_tokens(self, openai, mock_llmobs_writer, mock_tracer):
             )
         )
 
+    @pytest.mark.skipif(
+        parse_version(openai_module.version.VERSION) < (1, 66), reason="Response options only available openai >= 1.66"
+    )
+    def test_response_stream_incomplete(self, openai, mock_llmobs_writer, mock_tracer):
+        client = openai.OpenAI()
+        request_args = {
+            "model": "gpt-4o",
+            "max_output_tokens": 16,
+            "temperature": 0.1,
+            "stream": True,
+        }
+        with get_openai_vcr(subdirectory_name="v1").use_cassette("response_stream_incomplete.yaml"):
+            resp1 = client.responses.create(
+                input="Give me a multi paragraph narrative on the life of a car",
+                **request_args,
+            )
+            for chunk in resp1:
+                if hasattr(chunk, "response") and hasattr(chunk.response, "model"):
+                    resp_model = chunk.response.model
+        span = mock_tracer.pop_traces()[0][0]
+        assert mock_llmobs_writer.enqueue.call_count == 1
+        mock_llmobs_writer.enqueue.assert_called_with(
+            _expected_llmobs_llm_span_event(
+                span,
+                model_name=resp_model,
+                model_provider="openai",
+                input_messages=[
+                    {"content": "Give me a multi paragraph narrative on the life of a car", "role": "user"}
+                ],
+                output_messages=[
+                    {
+                        "role": "assistant",
+                        "content": "In the bustling city of Detroit, a sleek, metallic blue sedan rolled off the",
+                    }
+                ],
+                metadata={
+                    "max_output_tokens": 16,
+                    "temperature": 0.1,
+                    "stream": True,
+                    "top_p": 1.0,
+                    "tools": [],
+                    "tool_choice": "auto",
+                    "truncation": "disabled",
+                    "text": {"format": {"type": "text"}},
+                    "reasoning_tokens": 0,
+                },
+                token_metrics={"input_tokens": 0, "output_tokens": 0, "total_tokens": 0},
+                tags={"ml_app": "<ml-app-name>", "service": "tests.contrib.openai"},
+            )
+        )
+
     @pytest.mark.skipif(
         parse_version(openai_module.version.VERSION) < (1, 66), reason="Response options only available openai >= 1.66"
     )

-Original file line number
+Diff line change
@@ @@ -0,0 +1,4 @@ @@
 +---
 +fixes:
 +  - |
 +    LLM Observability: This fix resolves an issue where incomplete streamed responses returned from OpenAI responses API caused an index error with LLM Observability tracing.