Skip to content

Experimental: allow inline/anonymous TypedDicts #17457

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Jul 7, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions docs/source/command_line.rst
Original file line number Diff line number Diff line change
Expand Up @@ -1055,6 +1055,27 @@ List of currently incomplete/experimental features:
# Without PreciseTupleTypes: tuple[int, ...]
# With PreciseTupleTypes: tuple[()] | tuple[int] | tuple[int, int]

* ``NewGenericSyntax``: this feature enables support for syntax defined
by :pep:`695`. For example:

.. code-block:: python

class Container[T]: # defines a generic class
content: T

def first[T](items: list[T]) -> T: # defines a generic function
return items[0]

type Items[T] = list[tuple[T, T]] # defines a generic type alias

* ``InlineTypedDict``: this feature enables non-standard syntax for inline
:ref:`TypedDicts <typeddict>`, for example:

.. code-block:: python

def test_values() -> {"int": int, "str": str}:
return {"int": 42, "str": "test"}


Miscellaneous
*************
Expand Down
38 changes: 38 additions & 0 deletions docs/source/typed_dict.rst
Original file line number Diff line number Diff line change
Expand Up @@ -248,3 +248,41 @@ section of the docs has a full description with an example, but in short, you wi
need to give each TypedDict the same key where each value has a unique
:ref:`Literal type <literal_types>`. Then, check that key to distinguish
between your TypedDicts.

Inline TypedDict types
----------------------

.. note::

This is an experimental (non-standard) feature. Use
``--enable-incomplete-feature=InlineTypedDict`` to enable.

Sometimes you may want to define a complex nested JSON schema, or annotate
a one-off function that returns a TypedDict. In such cases it may be convenient
to use inline TypedDict syntax. For example:

.. code-block:: python

def test_values() -> {"int": int, "str": str}:
return {"int": 42, "str": "test"}

class Response(TypedDict):
status: int
msg: str
# Using inline syntax here avoids defining two additional TypedDicts.
content: {"items": list[{"key": str, "value": str}]}

Inline TypedDicts can also by used as targets of type aliases, but due to
ambiguity with a regular variables it is only allowed for (newer) explicit
type alias forms:

.. code-block:: python

from typing import TypeAlias

X = {"a": int, "b": int} # creates a variable with type dict[str, type[int]]
Y: TypeAlias = {"a": int, "b": int} # creates a type alias
type Z = {"a": int, "b": int} # same as above (Python 3.12+ only)

Also, due to incompatibility with runtime type-checking it is strongly recommended
to *not* use inline syntax in union types.
6 changes: 4 additions & 2 deletions mypy/checker.py
Original file line number Diff line number Diff line change
Expand Up @@ -2971,7 +2971,8 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None:
self.msg.annotation_in_unchecked_function(context=s)

def check_type_alias_rvalue(self, s: AssignmentStmt) -> None:
alias_type = self.expr_checker.accept(s.rvalue)
with self.msg.filter_errors():
alias_type = self.expr_checker.accept(s.rvalue)
self.store_type(s.lvalues[-1], alias_type)

def check_assignment(
Expand Down Expand Up @@ -5311,7 +5312,8 @@ def remove_capture_conflicts(self, type_map: TypeMap, inferred_types: dict[Var,
del type_map[expr]

def visit_type_alias_stmt(self, o: TypeAliasStmt) -> None:
self.expr_checker.accept(o.value)
with self.msg.filter_errors():
self.expr_checker.accept(o.value)

def make_fake_typeinfo(
self,
Expand Down
31 changes: 29 additions & 2 deletions mypy/exprtotype.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,11 @@

from mypy.fastparse import parse_type_string
from mypy.nodes import (
MISSING_FALLBACK,
BytesExpr,
CallExpr,
ComplexExpr,
DictExpr,
EllipsisExpr,
Expression,
FloatExpr,
Expand All @@ -29,9 +31,11 @@
AnyType,
CallableArgument,
EllipsisType,
Instance,
ProperType,
RawExpressionType,
Type,
TypedDictType,
TypeList,
TypeOfAny,
UnboundType,
Expand All @@ -55,7 +59,7 @@ def _extract_argument_name(expr: Expression) -> str | None:

def expr_to_unanalyzed_type(
expr: Expression,
options: Options | None = None,
options: Options,
allow_new_syntax: bool = False,
_parent: Expression | None = None,
allow_unpack: bool = False,
Expand All @@ -67,6 +71,8 @@ def expr_to_unanalyzed_type(

If allow_new_syntax is True, allow all type syntax independent of the target
Python version (used in stubs).

# TODO: a lot of code here is duplicated in fastparse.py, refactor this.
"""
# The `parent` parameter is used in recursive calls to provide context for
# understanding whether an CallableArgument is ok.
Expand Down Expand Up @@ -116,7 +122,7 @@ def expr_to_unanalyzed_type(
elif (
isinstance(expr, OpExpr)
and expr.op == "|"
and ((options and options.python_version >= (3, 10)) or allow_new_syntax)
and ((options.python_version >= (3, 10)) or allow_new_syntax)
):
return UnionType(
[
Expand Down Expand Up @@ -206,5 +212,26 @@ def expr_to_unanalyzed_type(
return UnpackType(
expr_to_unanalyzed_type(expr.expr, options, allow_new_syntax), from_star_syntax=True
)
elif isinstance(expr, DictExpr):
if not expr.items:
raise TypeTranslationError()
items: dict[str, Type] = {}
extra_items_from = []
for item_name, value in expr.items:
if not isinstance(item_name, StrExpr):
if item_name is None:
extra_items_from.append(
expr_to_unanalyzed_type(value, options, allow_new_syntax, expr)
)
continue
raise TypeTranslationError()
items[item_name.value] = expr_to_unanalyzed_type(
value, options, allow_new_syntax, expr
)
result = TypedDictType(
items, set(), Instance(MISSING_FALLBACK, ()), expr.line, expr.column
)
result.extra_items_from = extra_items_from
return result
else:
raise TypeTranslationError()
20 changes: 18 additions & 2 deletions mypy/fastparse.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
ARG_POS,
ARG_STAR,
ARG_STAR2,
MISSING_FALLBACK,
PARAM_SPEC_KIND,
TYPE_VAR_KIND,
TYPE_VAR_TUPLE_KIND,
Expand All @@ -42,7 +43,6 @@
EllipsisExpr,
Expression,
ExpressionStmt,
FakeInfo,
FloatExpr,
ForStmt,
FuncDef,
Expand Down Expand Up @@ -116,6 +116,7 @@
RawExpressionType,
TupleType,
Type,
TypedDictType,
TypeList,
TypeOfAny,
UnboundType,
Expand Down Expand Up @@ -190,7 +191,6 @@ def ast3_parse(

# There is no way to create reasonable fallbacks at this stage,
# they must be patched later.
MISSING_FALLBACK: Final = FakeInfo("fallback can't be filled out until semanal")
_dummy_fallback: Final = Instance(MISSING_FALLBACK, [], -1)

TYPE_IGNORE_PATTERN: Final = re.compile(r"[^#]*#\s*type:\s*ignore\s*(.*)")
Expand Down Expand Up @@ -2096,6 +2096,22 @@ def visit_Tuple(self, n: ast3.Tuple) -> Type:
column=self.convert_column(n.col_offset),
)

def visit_Dict(self, n: ast3.Dict) -> Type:
if not n.keys:
return self.invalid_type(n)
items: dict[str, Type] = {}
extra_items_from = []
for item_name, value in zip(n.keys, n.values):
if not isinstance(item_name, ast3.Constant) or not isinstance(item_name.value, str):
if item_name is None:
extra_items_from.append(self.visit(value))
continue
return self.invalid_type(n)
items[item_name.value] = self.visit(value)
result = TypedDictType(items, set(), _dummy_fallback, n.lineno, n.col_offset)
result.extra_items_from = extra_items_from
return result

# Attribute(expr value, identifier attr, expr_context ctx)
def visit_Attribute(self, n: Attribute) -> Type:
before_dot = self.visit(n.value)
Expand Down
1 change: 1 addition & 0 deletions mypy/message_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,7 @@ def with_additional_msg(self, info: str) -> ErrorMessage:
TYPEDDICT_KEY_MUST_BE_STRING_LITERAL: Final = ErrorMessage(
"Expected TypedDict key to be string literal"
)
TYPEDDICT_OVERRIDE_MERGE: Final = 'Overwriting TypedDict field "{}" while merging'
MALFORMED_ASSERT: Final = ErrorMessage("Assertion is always true, perhaps remove parentheses?")
DUPLICATE_TYPE_SIGNATURES: Final = ErrorMessage("Function has duplicate type signatures")
DESCRIPTOR_SET_NOT_CALLABLE: Final = ErrorMessage("{}.__set__ is not callable")
Expand Down
1 change: 1 addition & 0 deletions mypy/nodes.py
Original file line number Diff line number Diff line change
Expand Up @@ -3480,6 +3480,7 @@ def __getattribute__(self, attr: str) -> type:
VAR_NO_INFO: Final[TypeInfo] = FakeInfo("Var is lacking info")
CLASSDEF_NO_INFO: Final[TypeInfo] = FakeInfo("ClassDef is lacking info")
FUNC_NO_INFO: Final[TypeInfo] = FakeInfo("FuncBase for non-methods lack info")
MISSING_FALLBACK: Final = FakeInfo("fallback can't be filled out until semanal")


class TypeAlias(SymbolNode):
Expand Down
3 changes: 2 additions & 1 deletion mypy/options.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,8 @@ class BuildType:
UNPACK: Final = "Unpack"
PRECISE_TUPLE_TYPES: Final = "PreciseTupleTypes"
NEW_GENERIC_SYNTAX: Final = "NewGenericSyntax"
INCOMPLETE_FEATURES: Final = frozenset((PRECISE_TUPLE_TYPES, NEW_GENERIC_SYNTAX))
INLINE_TYPEDDICT: Final = "InlineTypedDict"
INCOMPLETE_FEATURES: Final = frozenset((PRECISE_TUPLE_TYPES, NEW_GENERIC_SYNTAX, INLINE_TYPEDDICT))
COMPLETE_FEATURES: Final = frozenset((TYPE_VAR_TUPLE, UNPACK))


Expand Down
15 changes: 3 additions & 12 deletions mypy/semanal_typeddict.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
from mypy.errorcodes import ErrorCode
from mypy.expandtype import expand_type
from mypy.exprtotype import TypeTranslationError, expr_to_unanalyzed_type
from mypy.message_registry import TYPEDDICT_OVERRIDE_MERGE
from mypy.messages import MessageBuilder
from mypy.nodes import (
ARG_NAMED,
Expand Down Expand Up @@ -216,7 +217,7 @@ def add_keys_and_types_from_base(
valid_items = self.map_items_to_base(valid_items, tvars, base_args)
for key in base_items:
if key in keys:
self.fail(f'Overwriting TypedDict field "{key}" while merging', ctx)
self.fail(TYPEDDICT_OVERRIDE_MERGE.format(key), ctx)
keys.extend(valid_items.keys())
types.extend(valid_items.values())
required_keys.update(base_typed_dict.required_keys)
Expand Down Expand Up @@ -505,17 +506,7 @@ def parse_typeddict_fields_with_types(
field_type_expr, self.options, self.api.is_stub_file
)
except TypeTranslationError:
if (
isinstance(field_type_expr, CallExpr)
and isinstance(field_type_expr.callee, RefExpr)
and field_type_expr.callee.fullname in TPDICT_NAMES
):
self.fail_typeddict_arg(
"Inline TypedDict types not supported; use assignment to define TypedDict",
field_type_expr,
)
else:
self.fail_typeddict_arg("Invalid field type", field_type_expr)
self.fail_typeddict_arg("Use dict literal for nested TypedDict", field_type_expr)
return [], [], False
analyzed = self.api.anal_type(
type,
Expand Down
55 changes: 48 additions & 7 deletions mypy/typeanal.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,11 @@
from mypy import errorcodes as codes, message_registry, nodes
from mypy.errorcodes import ErrorCode
from mypy.expandtype import expand_type
from mypy.message_registry import INVALID_PARAM_SPEC_LOCATION, INVALID_PARAM_SPEC_LOCATION_NOTE
from mypy.message_registry import (
INVALID_PARAM_SPEC_LOCATION,
INVALID_PARAM_SPEC_LOCATION_NOTE,
TYPEDDICT_OVERRIDE_MERGE,
)
from mypy.messages import (
MessageBuilder,
format_type,
Expand All @@ -25,6 +29,7 @@
ARG_POS,
ARG_STAR,
ARG_STAR2,
MISSING_FALLBACK,
SYMBOL_FUNCBASE_TYPES,
ArgKind,
Context,
Expand All @@ -43,7 +48,7 @@
check_arg_names,
get_nongen_builtins,
)
from mypy.options import Options
from mypy.options import INLINE_TYPEDDICT, Options
from mypy.plugin import AnalyzeTypeContext, Plugin, TypeAnalyzerPluginInterface
from mypy.semanal_shared import (
SemanticAnalyzerCoreInterface,
Expand Down Expand Up @@ -1220,10 +1225,45 @@ def visit_tuple_type(self, t: TupleType) -> Type:
return TupleType(self.anal_array(t.items, allow_unpack=True), fallback, t.line)

def visit_typeddict_type(self, t: TypedDictType) -> Type:
items = {
item_name: self.anal_type(item_type) for (item_name, item_type) in t.items.items()
}
return TypedDictType(items, set(t.required_keys), t.fallback)
req_keys = set()
items = {}
for item_name, item_type in t.items.items():
analyzed = self.anal_type(item_type, allow_required=True)
if isinstance(analyzed, RequiredType):
if analyzed.required:
req_keys.add(item_name)
analyzed = analyzed.item
else:
# Keys are required by default.
req_keys.add(item_name)
items[item_name] = analyzed
if t.fallback.type is MISSING_FALLBACK: # anonymous/inline TypedDict
if INLINE_TYPEDDICT not in self.options.enable_incomplete_feature:
self.fail(
"Inline TypedDict is experimental,"
" must be enabled with --enable-incomplete-feature=InlineTypedDict",
t,
)
required_keys = req_keys
fallback = self.named_type("typing._TypedDict")
for typ in t.extra_items_from:
analyzed = self.analyze_type(typ)
p_analyzed = get_proper_type(analyzed)
if not isinstance(p_analyzed, TypedDictType):
if not isinstance(p_analyzed, (AnyType, PlaceholderType)):
self.fail("Can only merge-in other TypedDict", t, code=codes.VALID_TYPE)
continue
for sub_item_name, sub_item_type in p_analyzed.items.items():
if sub_item_name in items:
self.fail(TYPEDDICT_OVERRIDE_MERGE.format(sub_item_name), t)
continue
items[sub_item_name] = sub_item_type
if sub_item_name in p_analyzed.required_keys:
req_keys.add(sub_item_name)
else:
required_keys = t.required_keys
fallback = t.fallback
return TypedDictType(items, required_keys, fallback, t.line, t.column)

def visit_raw_expression_type(self, t: RawExpressionType) -> Type:
# We should never see a bare Literal. We synthesize these raw literals
Expand Down Expand Up @@ -1761,11 +1801,12 @@ def anal_type(
allow_param_spec: bool = False,
allow_unpack: bool = False,
allow_ellipsis: bool = False,
allow_required: bool = False,
) -> Type:
if nested:
self.nesting_level += 1
old_allow_required = self.allow_required
self.allow_required = False
self.allow_required = allow_required
old_allow_ellipsis = self.allow_ellipsis
self.allow_ellipsis = allow_ellipsis
old_allow_unpack = self.allow_unpack
Expand Down
Loading
Loading