Support native JSON output and strict tool calls for Anthropic (#3457)

Co-authored-by: Douwe Maan <[email protected]>

Author: dsfaccini

docs/output.md

@@ -308,7 +308,7 @@ _(This example is complete, it can be run "as is")_
 
 #### Native Output
 
-Native Output mode uses a model's native "Structured Outputs" feature (aka "JSON Schema response format"), where the model is forced to only output text matching the provided JSON schema. Note that this is not supported by all models, and sometimes comes with restrictions. For example, Anthropic does not support this at all, and Gemini cannot use tools at the same time as structured output, and attempting to do so will result in an error.
+Native Output mode uses a model's native "Structured Outputs" feature (aka "JSON Schema response format"), where the model is forced to only output text matching the provided JSON schema. Note that this is not supported by all models, and sometimes comes with restrictions. For example, Gemini cannot use tools at the same time as structured output, and attempting to do so will result in an error.
 
 To use this mode, you can wrap the output type(s) in the [`NativeOutput`][pydantic_ai.output.NativeOutput] marker class that also lets you specify a `name` and `description` if the name and docstring of the type or function are not sufficient.
 

pydantic_ai_slim/pydantic_ai/_json_schema.py

@@ -15,6 +15,9 @@
 class JsonSchemaTransformer(ABC):
     """Walks a JSON schema, applying transformations to it at each level.
 
+    The transformer is called during a model's prepare_request() step to build the JSON schema
+    before it is sent to the model provider.
+
     Note: We may eventually want to rework tools to build the JSON schema from the type directly, using a subclass of
     pydantic.json_schema.GenerateJsonSchema, rather than making use of this machinery.
     """
@@ -30,8 +33,15 @@ def __init__(
         self.schema = schema
 
         self.strict = strict
-        self.is_strict_compatible = True  # Can be set to False by subclasses to set `strict` on `ToolDefinition` when set not set by user explicitly
+        """The `strict` parameter forces the conversion of the original JSON schema (`self.schema`) of a `ToolDefinition` or `OutputObjectDefinition` to a format supported by the model provider.
+
+        The "strict mode" offered by model providers ensures that the model's output adheres closely to the defined schema. However, not all model providers offer it, and their support for various schema features may differ. For example, a model provider's required schema may not support certain validation constraints like `minLength` or `pattern`.
+        """
+        self.is_strict_compatible = True
+        """Whether the schema is compatible with strict mode.
 
+        This value is used to set `ToolDefinition.strict` or `OutputObjectDefinition.strict` when their values are `None`.
+        """
         self.prefer_inlined_defs = prefer_inlined_defs
         self.simplify_nullable_unions = simplify_nullable_unions
 

pydantic_ai_slim/pydantic_ai/models/__init__.py

@@ -57,9 +57,6 @@
     Literal[
         'anthropic:claude-3-5-haiku-20241022',
         'anthropic:claude-3-5-haiku-latest',
-        'anthropic:claude-3-5-sonnet-20240620',
-        'anthropic:claude-3-5-sonnet-20241022',
-        'anthropic:claude-3-5-sonnet-latest',
         'anthropic:claude-3-7-sonnet-20250219',
         'anthropic:claude-3-7-sonnet-latest',
         'anthropic:claude-3-haiku-20240307',
@@ -72,6 +69,8 @@
         'anthropic:claude-opus-4-0',
         'anthropic:claude-opus-4-1-20250805',
         'anthropic:claude-opus-4-20250514',
+        'anthropic:claude-opus-4-5',
+        'anthropic:claude-opus-4-5-20251101',
         'anthropic:claude-sonnet-4-0',
         'anthropic:claude-sonnet-4-20250514',
         'anthropic:claude-sonnet-4-5',
@@ -100,6 +99,7 @@
         'bedrock:eu.anthropic.claude-haiku-4-5-20251001-v1:0',
         'bedrock:eu.anthropic.claude-sonnet-4-20250514-v1:0',
         'bedrock:eu.anthropic.claude-sonnet-4-5-20250929-v1:0',
+        'bedrock:global.anthropic.claude-opus-4-5-20251101-v1:0',
         'bedrock:meta.llama3-1-405b-instruct-v1:0',
         'bedrock:meta.llama3-1-70b-instruct-v1:0',
         'bedrock:meta.llama3-1-8b-instruct-v1:0',
@@ -380,7 +380,10 @@ async def request(
         model_settings: ModelSettings | None,
         model_request_parameters: ModelRequestParameters,
     ) -> ModelResponse:
-        """Make a request to the model."""
+        """Make a request to the model.
+
+        This is ultimately called by `pydantic_ai._agent_graph.ModelRequestNode._make_request(...)`.
+        """
         raise NotImplementedError()
 
     async def count_tokens(
@@ -987,23 +990,27 @@ def get_user_agent() -> str:
     return f'pydantic-ai/{__version__}'
 
 
-def _customize_tool_def(transformer: type[JsonSchemaTransformer], t: ToolDefinition):
-    schema_transformer = transformer(t.parameters_json_schema, strict=t.strict)
+def _customize_tool_def(transformer: type[JsonSchemaTransformer], tool_def: ToolDefinition):
+    """Customize the tool definition using the given transformer.
+
+    If the tool definition has `strict` set to None, the strictness will be inferred from the transformer.
+    """
+    schema_transformer = transformer(tool_def.parameters_json_schema, strict=tool_def.strict)
     parameters_json_schema = schema_transformer.walk()
     return replace(
-        t,
+        tool_def,
         parameters_json_schema=parameters_json_schema,
-        strict=schema_transformer.is_strict_compatible if t.strict is None else t.strict,
+        strict=schema_transformer.is_strict_compatible if tool_def.strict is None else tool_def.strict,
     )
 
 
-def _customize_output_object(transformer: type[JsonSchemaTransformer], o: OutputObjectDefinition):
-    schema_transformer = transformer(o.json_schema, strict=o.strict)
+def _customize_output_object(transformer: type[JsonSchemaTransformer], output_object: OutputObjectDefinition):
+    schema_transformer = transformer(output_object.json_schema, strict=output_object.strict)
     json_schema = schema_transformer.walk()
     return replace(
-        o,
+        output_object,
         json_schema=json_schema,
-        strict=schema_transformer.is_strict_compatible if o.strict is None else o.strict,
+        strict=schema_transformer.is_strict_compatible if output_object.strict is None else output_object.strict,
     )
 
 

pydantic_ai_slim/pydantic_ai/models/anthropic.py

@@ -78,6 +78,7 @@
         BetaContentBlockParam,
         BetaImageBlockParam,
         BetaInputJSONDelta,
+        BetaJSONOutputFormatParam,
         BetaMCPToolResultBlock,
         BetaMCPToolUseBlock,
         BetaMCPToolUseBlockParam,
@@ -225,8 +226,9 @@ def __init__(
             model_name: The name of the Anthropic model to use. List of model names available
                 [here](https://docs.anthropic.com/en/docs/about-claude/models).
             provider: The provider to use for the Anthropic API. Can be either the string 'anthropic' or an
-                instance of `Provider[AsyncAnthropicClient]`. If not provided, the other parameters will be used.
+                instance of `Provider[AsyncAnthropicClient]`. Defaults to 'anthropic'.
             profile: The model profile to use. Defaults to a profile picked by the provider based on the model name.
+                The default 'anthropic' provider will use the default `..profiles.anthropic.anthropic_model_profile`.
             settings: Default model settings for this model instance.
         """
         self._model_name = model_name
@@ -316,14 +318,26 @@ def prepare_request(
             and thinking.get('type') == 'enabled'
         ):
             if model_request_parameters.output_mode == 'auto':
-                model_request_parameters = replace(model_request_parameters, output_mode='prompted')
+                output_mode = 'native' if self.profile.supports_json_schema_output else 'prompted'
+                model_request_parameters = replace(model_request_parameters, output_mode=output_mode)
             elif (
                 model_request_parameters.output_mode == 'tool' and not model_request_parameters.allow_text_output
             ):  # pragma: no branch
                 # This would result in `tool_choice=required`, which Anthropic does not support with thinking.
+                suggested_output_type = 'NativeOutput' if self.profile.supports_json_schema_output else 'PromptedOutput'
                 raise UserError(
-                    'Anthropic does not support thinking and output tools at the same time. Use `output_type=PromptedOutput(...)` instead.'
+                    f'Anthropic does not support thinking and output tools at the same time. Use `output_type={suggested_output_type}(...)` instead.'
                 )
+
+        if model_request_parameters.output_mode == 'native':
+            assert model_request_parameters.output_object is not None
+            if model_request_parameters.output_object.strict is False:
+                raise UserError(
+                    'Setting `strict=False` on `output_type=NativeOutput(...)` is not allowed for Anthropic models.'
+                )
+            model_request_parameters = replace(
+                model_request_parameters, output_object=replace(model_request_parameters.output_object, strict=True)
+            )
         return super().prepare_request(model_settings, model_request_parameters)
 
     @overload
@@ -353,17 +367,22 @@ async def _messages_create(
         model_settings: AnthropicModelSettings,
         model_request_parameters: ModelRequestParameters,
     ) -> BetaMessage | AsyncStream[BetaRawMessageStreamEvent]:
-        # standalone function to make it easier to override
+        """Calls the Anthropic API to create a message.
+
+        This is the last step before sending the request to the API.
+        Most preprocessing has happened in `prepare_request()`.
+        """
         tools = self._get_tools(model_request_parameters, model_settings)
-        tools, mcp_servers, beta_features = self._add_builtin_tools(tools, model_request_parameters)
+        tools, mcp_servers, builtin_tool_betas = self._add_builtin_tools(tools, model_request_parameters)
 
         tool_choice = self._infer_tool_choice(tools, model_settings, model_request_parameters)
 
         system_prompt, anthropic_messages = await self._map_message(messages, model_request_parameters, model_settings)
         self._limit_cache_points(system_prompt, anthropic_messages, tools)
+        output_format = self._native_output_format(model_request_parameters)
+        betas, extra_headers = self._get_betas_and_extra_headers(tools, model_request_parameters, model_settings)
+        betas.update(builtin_tool_betas)
         try:
-            extra_headers = self._map_extra_headers(beta_features, model_settings)
-
             return await self.client.beta.messages.create(
                 max_tokens=model_settings.get('max_tokens', 4096),
                 system=system_prompt or OMIT,
@@ -372,6 +391,8 @@ async def _messages_create(
                 tools=tools or OMIT,
                 tool_choice=tool_choice or OMIT,
                 mcp_servers=mcp_servers or OMIT,
+                output_format=output_format or OMIT,
+                betas=sorted(betas) or OMIT,
                 stream=stream,
                 thinking=model_settings.get('anthropic_thinking', OMIT),
                 stop_sequences=model_settings.get('stop_sequences', OMIT),
@@ -389,6 +410,32 @@ async def _messages_create(
         except APIConnectionError as e:
             raise ModelAPIError(model_name=self.model_name, message=e.message) from e
 
+    def _get_betas_and_extra_headers(
+        self,
+        tools: list[BetaToolUnionParam],
+        model_request_parameters: ModelRequestParameters,
+        model_settings: AnthropicModelSettings,
+    ) -> tuple[set[str], dict[str, str]]:
+        """Prepare beta features list and extra headers for API request.
+
+        Handles merging custom `anthropic-beta` header from `extra_headers` into betas set
+        and ensuring `User-Agent` is set.
+        """
+        extra_headers = model_settings.get('extra_headers', {})
+        extra_headers.setdefault('User-Agent', get_user_agent())
+
+        betas: set[str] = set()
+
+        has_strict_tools = any(tool.get('strict') for tool in tools)
+
+        if has_strict_tools or model_request_parameters.output_mode == 'native':
+            betas.add('structured-outputs-2025-11-13')
+
+        if beta_header := extra_headers.pop('anthropic-beta', None):
+            betas.update({stripped_beta for beta in beta_header.split(',') if (stripped_beta := beta.strip())})
+
+        return betas, extra_headers
+
     async def _messages_count_tokens(
         self,
         messages: list[ModelMessage],
@@ -400,22 +447,25 @@ async def _messages_count_tokens(
 
         # standalone function to make it easier to override
         tools = self._get_tools(model_request_parameters, model_settings)
-        tools, mcp_servers, beta_features = self._add_builtin_tools(tools, model_request_parameters)
+        tools, mcp_servers, builtin_tool_betas = self._add_builtin_tools(tools, model_request_parameters)
 
         tool_choice = self._infer_tool_choice(tools, model_settings, model_request_parameters)
 
         system_prompt, anthropic_messages = await self._map_message(messages, model_request_parameters, model_settings)
         self._limit_cache_points(system_prompt, anthropic_messages, tools)
+        output_format = self._native_output_format(model_request_parameters)
+        betas, extra_headers = self._get_betas_and_extra_headers(tools, model_request_parameters, model_settings)
+        betas.update(builtin_tool_betas)
         try:
-            extra_headers = self._map_extra_headers(beta_features, model_settings)
-
             return await self.client.beta.messages.count_tokens(
                 system=system_prompt or OMIT,
                 messages=anthropic_messages,
                 model=self._model_name,
                 tools=tools or OMIT,
                 tool_choice=tool_choice or OMIT,
                 mcp_servers=mcp_servers or OMIT,
+                betas=sorted(betas) or OMIT,
+                output_format=output_format or OMIT,
                 thinking=model_settings.get('anthropic_thinking', OMIT),
                 timeout=model_settings.get('timeout', NOT_GIVEN),
                 extra_headers=extra_headers,
@@ -521,8 +571,8 @@ def _get_tools(
 
     def _add_builtin_tools(
         self, tools: list[BetaToolUnionParam], model_request_parameters: ModelRequestParameters
-    ) -> tuple[list[BetaToolUnionParam], list[BetaRequestMCPServerURLDefinitionParam], list[str]]:
-        beta_features: list[str] = []
+    ) -> tuple[list[BetaToolUnionParam], list[BetaRequestMCPServerURLDefinitionParam], set[str]]:
+        beta_features: set[str] = set()
         mcp_servers: list[BetaRequestMCPServerURLDefinitionParam] = []
         for tool in model_request_parameters.builtin_tools:
             if isinstance(tool, WebSearchTool):
@@ -539,7 +589,7 @@ def _add_builtin_tools(
                 )
             elif isinstance(tool, CodeExecutionTool):  # pragma: no branch
                 tools.append(BetaCodeExecutionTool20250522Param(name='code_execution', type='code_execution_20250522'))
-                beta_features.append('code-execution-2025-05-22')
+                beta_features.add('code-execution-2025-05-22')
             elif isinstance(tool, WebFetchTool):  # pragma: no branch
                 citations = BetaCitationsConfigParam(enabled=tool.enable_citations) if tool.enable_citations else None
                 tools.append(
@@ -553,14 +603,14 @@ def _add_builtin_tools(
                         max_content_tokens=tool.max_content_tokens,
                     )
                 )
-                beta_features.append('web-fetch-2025-09-10')
+                beta_features.add('web-fetch-2025-09-10')
             elif isinstance(tool, MemoryTool):  # pragma: no branch
                 if 'memory' not in model_request_parameters.tool_defs:
                     raise UserError("Built-in `MemoryTool` requires a 'memory' tool to be defined.")
                 # Replace the memory tool definition with the built-in memory tool
-                tools = [tool for tool in tools if tool['name'] != 'memory']
+                tools = [tool for tool in tools if tool.get('name') != 'memory']
                 tools.append(BetaMemoryTool20250818Param(name='memory', type='memory_20250818'))
-                beta_features.append('context-management-2025-06-27')
+                beta_features.add('context-management-2025-06-27')
             elif isinstance(tool, MCPServerTool) and tool.url:
                 mcp_server_url_definition_param = BetaRequestMCPServerURLDefinitionParam(
                     type='url',
@@ -575,7 +625,7 @@ def _add_builtin_tools(
                 if tool.authorization_token:  # pragma: no cover
                     mcp_server_url_definition_param['authorization_token'] = tool.authorization_token
                 mcp_servers.append(mcp_server_url_definition_param)
-                beta_features.append('mcp-client-2025-04-04')
+                beta_features.add('mcp-client-2025-04-04')
             else:  # pragma: no cover
                 raise UserError(
                     f'`{tool.__class__.__name__}` is not supported by `AnthropicModel`. If it should be, please file an issue.'
@@ -603,16 +653,6 @@ def _infer_tool_choice(
 
             return tool_choice
 
-    def _map_extra_headers(self, beta_features: list[str], model_settings: AnthropicModelSettings) -> dict[str, str]:
-        """Apply beta_features to extra_headers in model_settings."""
-        extra_headers = model_settings.get('extra_headers', {})
-        extra_headers.setdefault('User-Agent', get_user_agent())
-        if beta_features:
-            if 'anthropic-beta' in extra_headers:
-                beta_features.insert(0, extra_headers['anthropic-beta'])
-            extra_headers['anthropic-beta'] = ','.join(beta_features)
-        return extra_headers
-
     async def _map_message(  # noqa: C901
         self,
         messages: list[ModelMessage],
@@ -992,13 +1032,23 @@ async def _map_user_prompt(
                 else:
                     raise RuntimeError(f'Unsupported content type: {type(item)}')  # pragma: no cover
 
-    @staticmethod
-    def _map_tool_definition(f: ToolDefinition) -> BetaToolParam:
-        return {
+    def _map_tool_definition(self, f: ToolDefinition) -> BetaToolParam:
+        """Maps a `ToolDefinition` dataclass to an Anthropic `BetaToolParam` dictionary."""
+        tool_param: BetaToolParam = {
             'name': f.name,
             'description': f.description or '',
             'input_schema': f.parameters_json_schema,
         }
+        if f.strict and self.profile.supports_json_schema_output:
+            tool_param['strict'] = f.strict
+        return tool_param
+
+    @staticmethod
+    def _native_output_format(model_request_parameters: ModelRequestParameters) -> BetaJSONOutputFormatParam | None:
+        if model_request_parameters.output_mode != 'native':
+            return None
+        assert model_request_parameters.output_object is not None
+        return {'type': 'json_schema', 'schema': model_request_parameters.output_object.json_schema}
 
 
 def _map_usage(
@@ -1221,6 +1271,9 @@ def _map_server_tool_use_block(item: BetaServerToolUseBlock, provider_name: str)
         )
     elif item.name in ('bash_code_execution', 'text_editor_code_execution'):  # pragma: no cover
         raise NotImplementedError(f'Anthropic built-in tool {item.name!r} is not currently supported.')
+    elif item.name in ('tool_search_tool_regex', 'tool_search_tool_bm25'):  # pragma: no cover
+        # NOTE this is being implemented in https://github.com/pydantic/pydantic-ai/pull/3550
+        raise NotImplementedError(f'Anthropic built-in tool {item.name!r} is not currently supported.')
     else:
         assert_never(item.name)