Skip to content

fix(langchain): do not read from streamed chunks when there are none for chat model streams [backport 3.17] #15058

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Oct 28, 2025
Merged

fix(langchain): do not read from streamed chunks when there are none for chat model streams [backport 3.17] #15058

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions ddtrace/contrib/internal/langchain/patch.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -320,10 +320,13 @@ def _on_span_started(span: Span):
integration.record_instance(instance, span)

def _on_span_finished(span: Span, streamed_chunks):
joined_chunks = streamed_chunks[0]
for chunk in streamed_chunks[1:]:
joined_chunks += chunk # base message types support __add__ for concatenation
kwargs["_dd.identifying_params"] = instance._identifying_params
if len(streamed_chunks):
joined_chunks = streamed_chunks[0]
for chunk in streamed_chunks[1:]:
joined_chunks += chunk # base message types support __add__ for concatenation
else:
joined_chunks = []
integration.llmobs_set_tags(span, args=args, kwargs=kwargs, response=joined_chunks, operation="chat")

return shared_stream(
Expand Down
4 changes: 4 additions & 0 deletions releasenotes/notes/langchain-early-stream-end-fix-af9ca6475b3aaf00.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
---
fixes:
- |
langchain: Fixes an issue where streamed responses that end before the first chunk is received would result in an ``IndexError``.
14 changes: 14 additions & 0 deletions tests/contrib/langchain/test_langchain.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -539,3 +539,17 @@ def circumference_tool(radius: float) -> float:
)

calculator.invoke("2", config={"unserializable": object()})


@pytest.mark.snapshot(ignores=["meta.error.stack", "meta.error.message"])
def test_streamed_chat_model_with_no_output(langchain_openai, openai_url):
from openai import APITimeoutError

chat_model = langchain_openai.ChatOpenAI(base_url=openai_url, timeout=0.0001)

result = chat_model.stream("Hello, my name is")
try:
next(result)
except Exception as e:
if not isinstance(e, APITimeoutError):
assert False, f"Expected APITimeoutError, got {e}"
154 changes: 154 additions & 0 deletions tests/llmobs/llmobs_cassettes/openai/openai_chat_completions_post_0fdef9ab.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,154 @@
interactions:
- request:
body: '{"messages":[{"content":"Hello, my name is","role":"user"}],"model":"gpt-3.5-turbo","n":1,"stream":true,"temperature":0.7}'
headers:
? !!python/object/apply:multidict._multidict.istr
- Accept
: - application/json
? !!python/object/apply:multidict._multidict.istr
- Accept-Encoding
: - gzip, deflate
? !!python/object/apply:multidict._multidict.istr
- Connection
: - keep-alive
Content-Length:
- '122'
? !!python/object/apply:multidict._multidict.istr
- Content-Type
: - application/json
? !!python/object/apply:multidict._multidict.istr
- User-Agent
: - OpenAI/Python 1.109.1
? !!python/object/apply:multidict._multidict.istr
- X-Stainless-Arch
: - arm64
? !!python/object/apply:multidict._multidict.istr
- X-Stainless-Async
: - 'false'
? !!python/object/apply:multidict._multidict.istr
- X-Stainless-Lang
: - python
? !!python/object/apply:multidict._multidict.istr
- X-Stainless-OS
: - MacOS
? !!python/object/apply:multidict._multidict.istr
- X-Stainless-Package-Version
: - 1.109.1
? !!python/object/apply:multidict._multidict.istr
- X-Stainless-Runtime
: - CPython
? !!python/object/apply:multidict._multidict.istr
- X-Stainless-Runtime-Version
: - 3.10.13
? !!python/object/apply:multidict._multidict.istr
- x-stainless-read-timeout
: - '0.0001'
? !!python/object/apply:multidict._multidict.istr
- x-stainless-retry-count
: - '0'
method: POST
uri: https://api.openai.com/v1/chat/completions
response:
body:
string: 'data: {"id":"chatcmpl-CTDgKS9E25YxCQOhLhNmhfOPiznNk","object":"chat.completion.chunk","created":1761080140,"model":"gpt-3.5-turbo-0125","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"obfuscation":"dDINu"}


data: {"id":"chatcmpl-CTDgKS9E25YxCQOhLhNmhfOPiznNk","object":"chat.completion.chunk","created":1761080140,"model":"gpt-3.5-turbo-0125","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"Assistant"},"logprobs":null,"finish_reason":null}],"obfuscation":"1yL6DImc2cky67"}


data: {"id":"chatcmpl-CTDgKS9E25YxCQOhLhNmhfOPiznNk","object":"chat.completion.chunk","created":1761080140,"model":"gpt-3.5-turbo-0125","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"obfuscation":"kyn0dy"}


data: {"id":"chatcmpl-CTDgKS9E25YxCQOhLhNmhfOPiznNk","object":"chat.completion.chunk","created":1761080140,"model":"gpt-3.5-turbo-0125","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"
How"},"logprobs":null,"finish_reason":null}],"obfuscation":"zF8"}


data: {"id":"chatcmpl-CTDgKS9E25YxCQOhLhNmhfOPiznNk","object":"chat.completion.chunk","created":1761080140,"model":"gpt-3.5-turbo-0125","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"
can"},"logprobs":null,"finish_reason":null}],"obfuscation":"ZOA"}


data: {"id":"chatcmpl-CTDgKS9E25YxCQOhLhNmhfOPiznNk","object":"chat.completion.chunk","created":1761080140,"model":"gpt-3.5-turbo-0125","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"
I"},"logprobs":null,"finish_reason":null}],"obfuscation":"yPnQN"}


data: {"id":"chatcmpl-CTDgKS9E25YxCQOhLhNmhfOPiznNk","object":"chat.completion.chunk","created":1761080140,"model":"gpt-3.5-turbo-0125","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"
assist"},"logprobs":null,"finish_reason":null}],"obfuscation":""}


data: {"id":"chatcmpl-CTDgKS9E25YxCQOhLhNmhfOPiznNk","object":"chat.completion.chunk","created":1761080140,"model":"gpt-3.5-turbo-0125","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"
you"},"logprobs":null,"finish_reason":null}],"obfuscation":"nG8"}


data: {"id":"chatcmpl-CTDgKS9E25YxCQOhLhNmhfOPiznNk","object":"chat.completion.chunk","created":1761080140,"model":"gpt-3.5-turbo-0125","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"
today"},"logprobs":null,"finish_reason":null}],"obfuscation":"a"}


data: {"id":"chatcmpl-CTDgKS9E25YxCQOhLhNmhfOPiznNk","object":"chat.completion.chunk","created":1761080140,"model":"gpt-3.5-turbo-0125","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"?"},"logprobs":null,"finish_reason":null}],"obfuscation":"KxSued"}


data: {"id":"chatcmpl-CTDgKS9E25YxCQOhLhNmhfOPiznNk","object":"chat.completion.chunk","created":1761080140,"model":"gpt-3.5-turbo-0125","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"obfuscation":"Q"}


data: [DONE]


'
headers:
CF-RAY:
- 9923a83a5cd9081d-IAD
Connection:
- keep-alive
Content-Type:
- text/event-stream; charset=utf-8
Date:
- Tue, 21 Oct 2025 20:55:40 GMT
Server:
- cloudflare
Set-Cookie:
- __cf_bm=o7TNYeatXbBOlFxeMlhl.8fe7kXVRNTm_dL98zIje8M-1761080140-1.0.1.1-FRpu._KCnEk.aGZG5YQ75Od_Ucq8okx9WLNY3JjdbbK3P7mwxS21FIvTtyY6GlllpujKEYLWkFHG6VIIw4zZmwH0yDL04t_gvEzyjWwc1qc;
path=/; expires=Tue, 21-Oct-25 21:25:40 GMT; domain=.api.openai.com; HttpOnly;
Secure; SameSite=None
- _cfuvid=bZiyEgtQ9lNlCOG2DaVgkJLkXs33574MrsoVFl0iIf4-1761080140438-0.0.1.1-604800000;
path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None
Strict-Transport-Security:
- max-age=31536000; includeSubDomains; preload
Transfer-Encoding:
- chunked
X-Content-Type-Options:
- nosniff
access-control-expose-headers:
- X-Request-ID
alt-svc:
- h3=":443"; ma=86400
cf-cache-status:
- DYNAMIC
openai-organization:
- datadog-staging
openai-processing-ms:
- '151'
openai-project:
- proj_gt6TQZPRbZfoY2J9AQlEJMpd
openai-version:
- '2020-10-01'
x-envoy-upstream-service-time:
- '174'
x-openai-proxy-wasm:
- v0.1
x-ratelimit-limit-requests:
- '10000'
x-ratelimit-limit-tokens:
- '50000000'
x-ratelimit-remaining-requests:
- '9999'
x-ratelimit-remaining-tokens:
- '49999993'
x-ratelimit-reset-requests:
- 6ms
x-ratelimit-reset-tokens:
- 0s
x-request-id:
- req_c9c685c9da2e4684b4613cefd4af98e8
status:
code: 200
message: OK
version: 1
154 changes: 154 additions & 0 deletions tests/llmobs/llmobs_cassettes/openai/openai_chat_completions_post_eb4ba851.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,154 @@
interactions:
- request:
body: '{"messages":[{"content":"Hello, my name is","role":"user"}],"model":"gpt-3.5-turbo","stream":true}'
headers:
? !!python/object/apply:multidict._multidict.istr
- Accept
: - application/json
? !!python/object/apply:multidict._multidict.istr
- Accept-Encoding
: - gzip, deflate, zstd
? !!python/object/apply:multidict._multidict.istr
- Connection
: - keep-alive
Content-Length:
- '98'
? !!python/object/apply:multidict._multidict.istr
- Content-Type
: - application/json
? !!python/object/apply:multidict._multidict.istr
- User-Agent
: - OpenAI/Python 1.109.1
? !!python/object/apply:multidict._multidict.istr
- X-Stainless-Arch
: - arm64
? !!python/object/apply:multidict._multidict.istr
- X-Stainless-Async
: - 'false'
? !!python/object/apply:multidict._multidict.istr
- X-Stainless-Lang
: - python
? !!python/object/apply:multidict._multidict.istr
- X-Stainless-OS
: - MacOS
? !!python/object/apply:multidict._multidict.istr
- X-Stainless-Package-Version
: - 1.109.1
? !!python/object/apply:multidict._multidict.istr
- X-Stainless-Runtime
: - CPython
? !!python/object/apply:multidict._multidict.istr
- X-Stainless-Runtime-Version
: - 3.10.13
? !!python/object/apply:multidict._multidict.istr
- x-stainless-read-timeout
: - '0.0001'
? !!python/object/apply:multidict._multidict.istr
- x-stainless-retry-count
: - '0'
method: POST
uri: https://api.openai.com/v1/chat/completions
response:
body:
string: 'data: {"id":"chatcmpl-CTDgOzn7iu7WvRbPx6d7DKv4jisZ1","object":"chat.completion.chunk","created":1761080144,"model":"gpt-3.5-turbo-0125","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"obfuscation":"GHVuj"}


data: {"id":"chatcmpl-CTDgOzn7iu7WvRbPx6d7DKv4jisZ1","object":"chat.completion.chunk","created":1761080144,"model":"gpt-3.5-turbo-0125","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"Assistant"},"logprobs":null,"finish_reason":null}],"obfuscation":"x2EMmcLlO3ecwG"}


data: {"id":"chatcmpl-CTDgOzn7iu7WvRbPx6d7DKv4jisZ1","object":"chat.completion.chunk","created":1761080144,"model":"gpt-3.5-turbo-0125","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"obfuscation":"TQ802v"}


data: {"id":"chatcmpl-CTDgOzn7iu7WvRbPx6d7DKv4jisZ1","object":"chat.completion.chunk","created":1761080144,"model":"gpt-3.5-turbo-0125","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"
How"},"logprobs":null,"finish_reason":null}],"obfuscation":"XKe"}


data: {"id":"chatcmpl-CTDgOzn7iu7WvRbPx6d7DKv4jisZ1","object":"chat.completion.chunk","created":1761080144,"model":"gpt-3.5-turbo-0125","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"
can"},"logprobs":null,"finish_reason":null}],"obfuscation":"kcL"}


data: {"id":"chatcmpl-CTDgOzn7iu7WvRbPx6d7DKv4jisZ1","object":"chat.completion.chunk","created":1761080144,"model":"gpt-3.5-turbo-0125","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"
I"},"logprobs":null,"finish_reason":null}],"obfuscation":"iYfTx"}


data: {"id":"chatcmpl-CTDgOzn7iu7WvRbPx6d7DKv4jisZ1","object":"chat.completion.chunk","created":1761080144,"model":"gpt-3.5-turbo-0125","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"
assist"},"logprobs":null,"finish_reason":null}],"obfuscation":""}


data: {"id":"chatcmpl-CTDgOzn7iu7WvRbPx6d7DKv4jisZ1","object":"chat.completion.chunk","created":1761080144,"model":"gpt-3.5-turbo-0125","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"
you"},"logprobs":null,"finish_reason":null}],"obfuscation":"3qp"}


data: {"id":"chatcmpl-CTDgOzn7iu7WvRbPx6d7DKv4jisZ1","object":"chat.completion.chunk","created":1761080144,"model":"gpt-3.5-turbo-0125","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"
today"},"logprobs":null,"finish_reason":null}],"obfuscation":"Q"}


data: {"id":"chatcmpl-CTDgOzn7iu7WvRbPx6d7DKv4jisZ1","object":"chat.completion.chunk","created":1761080144,"model":"gpt-3.5-turbo-0125","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"?"},"logprobs":null,"finish_reason":null}],"obfuscation":"rxGZdS"}


data: {"id":"chatcmpl-CTDgOzn7iu7WvRbPx6d7DKv4jisZ1","object":"chat.completion.chunk","created":1761080144,"model":"gpt-3.5-turbo-0125","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"obfuscation":"q"}


data: [DONE]


'
headers:
CF-RAY:
- 9923a8557c58310c-IAD
Connection:
- keep-alive
Content-Type:
- text/event-stream; charset=utf-8
Date:
- Tue, 21 Oct 2025 20:55:44 GMT
Server:
- cloudflare
Set-Cookie:
- __cf_bm=BNW6cRDCKHu3RmqDR.YwTVZBl0oaF1KmyP7rFJOVJh8-1761080144-1.0.1.1-Em.1pj_MusLdRZAzfxs.tpO51tHRIUprKfWcjhgW3dWoSXLap2PB6YRiem.DU.MG8NVyMIQkwW6W_JUxa_NnHFenF.ejDJXGieRs0VvP6cs;
path=/; expires=Tue, 21-Oct-25 21:25:44 GMT; domain=.api.openai.com; HttpOnly;
Secure; SameSite=None
- _cfuvid=ue2PaSkW.MPXEVwGniK0_bY41_Ri3BTQdhJbM_FzhWo-1761080144466-0.0.1.1-604800000;
path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None
Strict-Transport-Security:
- max-age=31536000; includeSubDomains; preload
Transfer-Encoding:
- chunked
X-Content-Type-Options:
- nosniff
access-control-expose-headers:
- X-Request-ID
alt-svc:
- h3=":443"; ma=86400
cf-cache-status:
- DYNAMIC
openai-organization:
- datadog-staging
openai-processing-ms:
- '134'
openai-project:
- proj_gt6TQZPRbZfoY2J9AQlEJMpd
openai-version:
- '2020-10-01'
x-envoy-upstream-service-time:
- '151'
x-openai-proxy-wasm:
- v0.1
x-ratelimit-limit-requests:
- '10000'
x-ratelimit-limit-tokens:
- '50000000'
x-ratelimit-remaining-requests:
- '9999'
x-ratelimit-remaining-tokens:
- '49999993'
x-ratelimit-reset-requests:
- 6ms
x-ratelimit-reset-tokens:
- 0s
x-request-id:
- req_396d9ab42a15454bbc29de5853e2bad7
status:
code: 200
message: OK
version: 1
Loading
Loading