diff --git a/CHANGELOG.md b/CHANGELOG.md index 871cf2b7..d7f1cace 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ - Support for `grpcio` stubs generation - Allow `mypy_protobuf.py` to be run directly as a script +- Add support for proto's [`well_known_types`](https://developers.google.com/protocol-buffers/docs/reference/python-generated#wkt) ## 1.24 diff --git a/generated/google/protobuf/duration_pb2.pyi.expected b/generated/google/protobuf/duration_pb2.pyi.expected new file mode 100644 index 00000000..973efdd7 --- /dev/null +++ b/generated/google/protobuf/duration_pb2.pyi.expected @@ -0,0 +1,46 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file +""" +from google.protobuf.descriptor import ( + Descriptor as google___protobuf___descriptor___Descriptor, + FileDescriptor as google___protobuf___descriptor___FileDescriptor, +) + +from google.protobuf.internal.well_known_types import ( + Duration as google___protobuf___internal___well_known_types___Duration, +) + +from google.protobuf.message import ( + Message as google___protobuf___message___Message, +) + +from typing import ( + Optional as typing___Optional, +) + +from typing_extensions import ( + Literal as typing_extensions___Literal, +) + + +builtin___bool = bool +builtin___bytes = bytes +builtin___float = float +builtin___int = int + + +DESCRIPTOR: google___protobuf___descriptor___FileDescriptor = ... + +class Duration(google___protobuf___message___Message, google___protobuf___internal___well_known_types___Duration): + DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... + seconds: builtin___int = ... + nanos: builtin___int = ... + + def __init__(self, + *, + seconds : typing___Optional[builtin___int] = None, + nanos : typing___Optional[builtin___int] = None, + ) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"nanos",b"nanos",u"seconds",b"seconds"]) -> None: ... +type___Duration = Duration diff --git a/generated/google/protobuf/empty_pb2.pyi.expected b/generated/google/protobuf/empty_pb2.pyi.expected new file mode 100644 index 00000000..4fc287d7 --- /dev/null +++ b/generated/google/protobuf/empty_pb2.pyi.expected @@ -0,0 +1,22 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file +""" +from google.protobuf.descriptor import ( + Descriptor as google___protobuf___descriptor___Descriptor, + FileDescriptor as google___protobuf___descriptor___FileDescriptor, +) + +from google.protobuf.message import ( + Message as google___protobuf___message___Message, +) + + +DESCRIPTOR: google___protobuf___descriptor___FileDescriptor = ... + +class Empty(google___protobuf___message___Message): + DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... + + def __init__(self, + ) -> None: ... +type___Empty = Empty diff --git a/proto/google/protobuf/duration.proto b/proto/google/protobuf/duration.proto new file mode 100644 index 00000000..99cb102c --- /dev/null +++ b/proto/google/protobuf/duration.proto @@ -0,0 +1,116 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option cc_enable_arenas = true; +option go_package = "github.com/golang/protobuf/ptypes/duration"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "DurationProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; + +// A Duration represents a signed, fixed-length span of time represented +// as a count of seconds and fractions of seconds at nanosecond +// resolution. It is independent of any calendar and concepts like "day" +// or "month". It is related to Timestamp in that the difference between +// two Timestamp values is a Duration and it can be added or subtracted +// from a Timestamp. Range is approximately +-10,000 years. +// +// # Examples +// +// Example 1: Compute Duration from two Timestamps in pseudo code. +// +// Timestamp start = ...; +// Timestamp end = ...; +// Duration duration = ...; +// +// duration.seconds = end.seconds - start.seconds; +// duration.nanos = end.nanos - start.nanos; +// +// if (duration.seconds < 0 && duration.nanos > 0) { +// duration.seconds += 1; +// duration.nanos -= 1000000000; +// } else if (duration.seconds > 0 && duration.nanos < 0) { +// duration.seconds -= 1; +// duration.nanos += 1000000000; +// } +// +// Example 2: Compute Timestamp from Timestamp + Duration in pseudo code. +// +// Timestamp start = ...; +// Duration duration = ...; +// Timestamp end = ...; +// +// end.seconds = start.seconds + duration.seconds; +// end.nanos = start.nanos + duration.nanos; +// +// if (end.nanos < 0) { +// end.seconds -= 1; +// end.nanos += 1000000000; +// } else if (end.nanos >= 1000000000) { +// end.seconds += 1; +// end.nanos -= 1000000000; +// } +// +// Example 3: Compute Duration from datetime.timedelta in Python. +// +// td = datetime.timedelta(days=3, minutes=10) +// duration = Duration() +// duration.FromTimedelta(td) +// +// # JSON Mapping +// +// In JSON format, the Duration type is encoded as a string rather than an +// object, where the string ends in the suffix "s" (indicating seconds) and +// is preceded by the number of seconds, with nanoseconds expressed as +// fractional seconds. For example, 3 seconds with 0 nanoseconds should be +// encoded in JSON format as "3s", while 3 seconds and 1 nanosecond should +// be expressed in JSON format as "3.000000001s", and 3 seconds and 1 +// microsecond should be expressed in JSON format as "3.000001s". +// +// +message Duration { + // Signed seconds of the span of time. Must be from -315,576,000,000 + // to +315,576,000,000 inclusive. Note: these bounds are computed from: + // 60 sec/min * 60 min/hr * 24 hr/day * 365.25 days/year * 10000 years + int64 seconds = 1; + + // Signed fractions of a second at nanosecond resolution of the span + // of time. Durations less than one second are represented with a 0 + // `seconds` field and a positive or negative `nanos` field. For durations + // of one second or more, a non-zero value for the `nanos` field must be + // of the same sign as the `seconds` field. Must be from -999,999,999 + // to +999,999,999 inclusive. + int32 nanos = 2; +} diff --git a/python/mypy_protobuf.py b/python/mypy_protobuf.py index 85f0f964..27c69b6d 100755 --- a/python/mypy_protobuf.py +++ b/python/mypy_protobuf.py @@ -11,6 +11,7 @@ import google.protobuf.descriptor_pb2 as d import six from google.protobuf.compiler import plugin_pb2 as plugin_pb2 +from google.protobuf.internal.well_known_types import WKTBASES MYPY = False if MYPY: @@ -257,7 +258,21 @@ def write_messages(self, messages, prefix): for desc in [m for m in messages if m.name not in PYTHON_RESERVED]: self.locals.add(desc.name) qualified_name = prefix + desc.name - l("class {}({}):", desc.name, message_class) + + # Reproduce some hardcoded logic from the protobuf implementation - where + # some specific "well_known_types" generated protos to have additional + # base classes + addl_base = u"" + if self.fd.package + "." + desc.name in WKTBASES: + # chop off the .proto - and import the well known type + # eg `from google.protobuf.duration import Duration` + well_known_type = WKTBASES[self.fd.package + "." + desc.name] + addl_base = ", " + self._import( + "google.protobuf.internal.well_known_types", + well_known_type.__name__, + ) + + l("class {}({}{}):", desc.name, message_class, addl_base) with self._indent(): l( "DESCRIPTOR: {} = ...", diff --git a/run_test.sh b/run_test.sh index dcb8d56e..126e88ca 100755 --- a/run_test.sh +++ b/run_test.sh @@ -112,5 +112,5 @@ find generated -type f -not \( -name "*.expected" -or -name "__init__.py" \) -de python --version py.test --version if [[ $PY_VER_UNIT_TESTS =~ ^2.* ]]; then IGNORE="--ignore=test/test_grpc_usage.py"; else IGNORE=""; fi - PYTHONPATH=generated py.test --ignore=generated $IGNORE + PYTHONPATH=generated py.test --ignore=generated $IGNORE -v ) diff --git a/test/test_generated_mypy.py b/test/test_generated_mypy.py index dece4528..a5d067aa 100644 --- a/test/test_generated_mypy.py +++ b/test/test_generated_mypy.py @@ -76,10 +76,12 @@ def compare_pyi_to_expected(output_path): def test_generate_mypy_matches(): # type: () -> None - proto_files = glob.glob("proto/testproto/*.proto") + glob.glob( - "proto/testproto/*/*.proto" + proto_files = ( + glob.glob("proto/testproto/*.proto") + + glob.glob("proto/testproto/*/*.proto") + + glob.glob("proto/google/protobuf/*.proto") ) - assert len(proto_files) == 12 # Just a sanity check that all the files show up + assert len(proto_files) == 13 # Just a sanity check that all the files show up failures = [] for fn in proto_files: diff --git a/test_negative/negative.py b/test_negative/negative.py index 2613c6ba..764f0276 100644 --- a/test_negative/negative.py +++ b/test_negative/negative.py @@ -87,6 +87,9 @@ # changes to typeshed a few months after the 1.24 release # See https://github.com/python/typeshed/pull/4833 _ = s.Extensions["foo"] # E:2.7 E:3.5 +# TODO - these will give errors once again once we are able to +# revert https://github.com/python/typeshed/pull/4833 later on +# a few months after 1.24 releases _ = s.Extensions[SeparateFileExtension.ext] _ = SeparateFileExtension.ext in s.Extensions del s.Extensions[SeparateFileExtension.ext] diff --git a/test_negative/output.expected.2.7 b/test_negative/output.expected.2.7 index ee3d7c59..b48b9b00 100644 --- a/test_negative/output.expected.2.7 +++ b/test_negative/output.expected.2.7 @@ -30,26 +30,26 @@ test_negative/negative.py:89: error: No overload variant of "__getitem__" of "_E test_negative/negative.py:89: note: Possible overload variants: test_negative/negative.py:89: note: def [_ExtenderMessageT <: Message] __getitem__(self, _ExtensionFieldDescriptor[Simple1, _ExtenderMessageT]) -> _ExtenderMessageT test_negative/negative.py:89: note: def __getitem__(self, FieldDescriptor) -> Any -test_negative/negative.py:99: error: Incompatible types in assignment (expression has type "int", variable has type "_ExtensionFieldDescriptor[Simple1, Any]") -test_negative/negative.py:103: error: Incompatible types in assignment (expression has type "Union[Literal[u'b_oneof_1'], Literal[u'b_oneof_2']]", variable has type "Union[Literal[u'a_oneof_1'], Literal[u'a_oneof_2'], Literal[u'outer_message_in_oneof'], Literal[u'outer_enum_in_oneof'], Literal[u'inner_enum_in_oneof']]") -test_negative/negative.py:106: error: "Descriptor" has no attribute "Garbage" -test_negative/negative.py:107: error: "Descriptor" has no attribute "Garbage" -test_negative/negative.py:110: error: "EnumDescriptor" has no attribute "Garbage" -test_negative/negative.py:113: error: "FileDescriptor" has no attribute "Garbage" -test_negative/negative.py:120: error: "OuterEnumValue" has no attribute "FOO" -test_negative/negative.py:124: error: Argument 1 to "Name" of "_EnumTypeWrapper" has incompatible type "int"; expected "OuterEnumValue" -test_negative/negative.py:126: error: Argument 1 to "Name" of "_EnumTypeWrapper" has incompatible type "InnerEnumValue"; expected "OuterEnumValue" -test_negative/negative.py:128: error: Argument 1 to "Name" of "_EnumTypeWrapper" has incompatible type "InnerEnumValue"; expected "OuterEnumValue" -test_negative/negative.py:130: error: Argument 1 to "Name" of "_EnumTypeWrapper" has incompatible type "InnerEnumValue"; expected "OuterEnumValue" -test_negative/negative.py:134: error: "ScalarMap[int, unicode]" has no attribute "get_or_create" -test_negative/negative.py:136: error: No overload variant of "get" of "Mapping" matches argument type "str" -test_negative/negative.py:136: note: Possible overload variant: -test_negative/negative.py:136: note: def get(self, k: int) -> Optional[unicode] -test_negative/negative.py:136: note: <1 more non-matching overload not shown> -test_negative/negative.py:137: error: No overload variant of "get" of "Mapping" matches argument type "str" -test_negative/negative.py:137: note: Possible overload variant: -test_negative/negative.py:137: note: def get(self, k: int) -> Optional[OuterMessage3] -test_negative/negative.py:137: note: <1 more non-matching overload not shown> -test_negative/negative.py:140: error: Incompatible types in assignment (expression has type "Optional[unicode]", variable has type "int") -test_negative/negative.py:141: error: Incompatible types in assignment (expression has type "Optional[OuterMessage3]", variable has type "int") -Found 41 errors in 1 file (checked 16 source files) +test_negative/negative.py:102: error: Incompatible types in assignment (expression has type "int", variable has type "_ExtensionFieldDescriptor[Simple1, Any]") +test_negative/negative.py:106: error: Incompatible types in assignment (expression has type "Union[Literal[u'b_oneof_1'], Literal[u'b_oneof_2']]", variable has type "Union[Literal[u'a_oneof_1'], Literal[u'a_oneof_2'], Literal[u'outer_message_in_oneof'], Literal[u'outer_enum_in_oneof'], Literal[u'inner_enum_in_oneof']]") +test_negative/negative.py:109: error: "Descriptor" has no attribute "Garbage" +test_negative/negative.py:110: error: "Descriptor" has no attribute "Garbage" +test_negative/negative.py:113: error: "EnumDescriptor" has no attribute "Garbage" +test_negative/negative.py:116: error: "FileDescriptor" has no attribute "Garbage" +test_negative/negative.py:123: error: "OuterEnumValue" has no attribute "FOO" +test_negative/negative.py:127: error: Argument 1 to "Name" of "_EnumTypeWrapper" has incompatible type "int"; expected "OuterEnumValue" +test_negative/negative.py:129: error: Argument 1 to "Name" of "_EnumTypeWrapper" has incompatible type "InnerEnumValue"; expected "OuterEnumValue" +test_negative/negative.py:131: error: Argument 1 to "Name" of "_EnumTypeWrapper" has incompatible type "InnerEnumValue"; expected "OuterEnumValue" +test_negative/negative.py:133: error: Argument 1 to "Name" of "_EnumTypeWrapper" has incompatible type "InnerEnumValue"; expected "OuterEnumValue" +test_negative/negative.py:137: error: "ScalarMap[int, unicode]" has no attribute "get_or_create" +test_negative/negative.py:139: error: No overload variant of "get" of "Mapping" matches argument type "str" +test_negative/negative.py:139: note: Possible overload variant: +test_negative/negative.py:139: note: def get(self, k: int) -> Optional[unicode] +test_negative/negative.py:139: note: <1 more non-matching overload not shown> +test_negative/negative.py:140: error: No overload variant of "get" of "Mapping" matches argument type "str" +test_negative/negative.py:140: note: Possible overload variant: +test_negative/negative.py:140: note: def get(self, k: int) -> Optional[OuterMessage3] +test_negative/negative.py:140: note: <1 more non-matching overload not shown> +test_negative/negative.py:143: error: Incompatible types in assignment (expression has type "Optional[unicode]", variable has type "int") +test_negative/negative.py:144: error: Incompatible types in assignment (expression has type "Optional[OuterMessage3]", variable has type "int") +Found 41 errors in 1 file (checked 17 source files) diff --git a/test_negative/output.expected.3.5 b/test_negative/output.expected.3.5 index 60ee6a41..2c382b61 100644 --- a/test_negative/output.expected.3.5 +++ b/test_negative/output.expected.3.5 @@ -52,26 +52,26 @@ test_negative/negative.py:89: error: No overload variant of "__getitem__" of "_E test_negative/negative.py:89: note: Possible overload variants: test_negative/negative.py:89: note: def [_ExtenderMessageT <: Message] __getitem__(self, _ExtensionFieldDescriptor[Simple1, _ExtenderMessageT]) -> _ExtenderMessageT test_negative/negative.py:89: note: def __getitem__(self, FieldDescriptor) -> Any -test_negative/negative.py:99: error: Incompatible types in assignment (expression has type "int", variable has type "_ExtensionFieldDescriptor[Simple1, Any]") -test_negative/negative.py:103: error: Incompatible types in assignment (expression has type "Union[Literal['b_oneof_1'], Literal['b_oneof_2']]", variable has type "Union[Literal['a_oneof_1'], Literal['a_oneof_2'], Literal['outer_message_in_oneof'], Literal['outer_enum_in_oneof'], Literal['inner_enum_in_oneof']]") -test_negative/negative.py:106: error: "Descriptor" has no attribute "Garbage" -test_negative/negative.py:107: error: "Descriptor" has no attribute "Garbage" -test_negative/negative.py:110: error: "EnumDescriptor" has no attribute "Garbage" -test_negative/negative.py:113: error: "FileDescriptor" has no attribute "Garbage" -test_negative/negative.py:120: error: "OuterEnumValue" has no attribute "FOO" -test_negative/negative.py:124: error: Argument 1 to "Name" of "_EnumTypeWrapper" has incompatible type "int"; expected "OuterEnumValue" -test_negative/negative.py:126: error: Argument 1 to "Name" of "_EnumTypeWrapper" has incompatible type "InnerEnumValue"; expected "OuterEnumValue" -test_negative/negative.py:128: error: Argument 1 to "Name" of "_EnumTypeWrapper" has incompatible type "InnerEnumValue"; expected "OuterEnumValue" -test_negative/negative.py:130: error: Argument 1 to "Name" of "_EnumTypeWrapper" has incompatible type "InnerEnumValue"; expected "OuterEnumValue" -test_negative/negative.py:134: error: "ScalarMap[int, str]" has no attribute "get_or_create" -test_negative/negative.py:136: error: No overload variant of "get" of "Mapping" matches argument type "str" -test_negative/negative.py:136: note: Possible overload variant: -test_negative/negative.py:136: note: def get(self, key: int) -> Optional[str] -test_negative/negative.py:136: note: <1 more non-matching overload not shown> -test_negative/negative.py:137: error: No overload variant of "get" of "Mapping" matches argument type "str" -test_negative/negative.py:137: note: Possible overload variant: -test_negative/negative.py:137: note: def get(self, key: int) -> Optional[OuterMessage3] -test_negative/negative.py:137: note: <1 more non-matching overload not shown> -test_negative/negative.py:140: error: Incompatible types in assignment (expression has type "Optional[str]", variable has type "int") -test_negative/negative.py:141: error: Incompatible types in assignment (expression has type "Optional[OuterMessage3]", variable has type "int") -Found 57 errors in 2 files (checked 28 source files) +test_negative/negative.py:102: error: Incompatible types in assignment (expression has type "int", variable has type "_ExtensionFieldDescriptor[Simple1, Any]") +test_negative/negative.py:106: error: Incompatible types in assignment (expression has type "Union[Literal['b_oneof_1'], Literal['b_oneof_2']]", variable has type "Union[Literal['a_oneof_1'], Literal['a_oneof_2'], Literal['outer_message_in_oneof'], Literal['outer_enum_in_oneof'], Literal['inner_enum_in_oneof']]") +test_negative/negative.py:109: error: "Descriptor" has no attribute "Garbage" +test_negative/negative.py:110: error: "Descriptor" has no attribute "Garbage" +test_negative/negative.py:113: error: "EnumDescriptor" has no attribute "Garbage" +test_negative/negative.py:116: error: "FileDescriptor" has no attribute "Garbage" +test_negative/negative.py:123: error: "OuterEnumValue" has no attribute "FOO" +test_negative/negative.py:127: error: Argument 1 to "Name" of "_EnumTypeWrapper" has incompatible type "int"; expected "OuterEnumValue" +test_negative/negative.py:129: error: Argument 1 to "Name" of "_EnumTypeWrapper" has incompatible type "InnerEnumValue"; expected "OuterEnumValue" +test_negative/negative.py:131: error: Argument 1 to "Name" of "_EnumTypeWrapper" has incompatible type "InnerEnumValue"; expected "OuterEnumValue" +test_negative/negative.py:133: error: Argument 1 to "Name" of "_EnumTypeWrapper" has incompatible type "InnerEnumValue"; expected "OuterEnumValue" +test_negative/negative.py:137: error: "ScalarMap[int, str]" has no attribute "get_or_create" +test_negative/negative.py:139: error: No overload variant of "get" of "Mapping" matches argument type "str" +test_negative/negative.py:139: note: Possible overload variant: +test_negative/negative.py:139: note: def get(self, key: int) -> Optional[str] +test_negative/negative.py:139: note: <1 more non-matching overload not shown> +test_negative/negative.py:140: error: No overload variant of "get" of "Mapping" matches argument type "str" +test_negative/negative.py:140: note: Possible overload variant: +test_negative/negative.py:140: note: def get(self, key: int) -> Optional[OuterMessage3] +test_negative/negative.py:140: note: <1 more non-matching overload not shown> +test_negative/negative.py:143: error: Incompatible types in assignment (expression has type "Optional[str]", variable has type "int") +test_negative/negative.py:144: error: Incompatible types in assignment (expression has type "Optional[OuterMessage3]", variable has type "int") +Found 57 errors in 2 files (checked 29 source files)