Skip to content

Commit 0b8c464

Browse files
nipunn1313nipunn1313
authored
Add support for well_known_types (#163)
Some google.protobuf types are hard coded within the protobuf compiler to have additional base classes. Add those base classes here and confirm that things work in typeshed.
1 parent f83001d commit 0b8c464

File tree

10 files changed

+256
-51
lines changed

10 files changed

+256
-51
lines changed

‎CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
- Support for `grpcio` stubs generation
44
- Allow `mypy_protobuf.py` to be run directly as a script
5+
- Add support for proto's [`well_known_types`](https://developers.google.com/protocol-buffers/docs/reference/python-generated#wkt)
56

67
## 1.24
78

‎generated/google/protobuf/duration_pb2.pyi.expected

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
"""
2+
@generated by mypy-protobuf. Do not edit manually!
3+
isort:skip_file
4+
"""
5+
from google.protobuf.descriptor import (
6+
Descriptor as google___protobuf___descriptor___Descriptor,
7+
FileDescriptor as google___protobuf___descriptor___FileDescriptor,
8+
)
9+
10+
from google.protobuf.internal.well_known_types import (
11+
Duration as google___protobuf___internal___well_known_types___Duration,
12+
)
13+
14+
from google.protobuf.message import (
15+
Message as google___protobuf___message___Message,
16+
)
17+
18+
from typing import (
19+
Optional as typing___Optional,
20+
)
21+
22+
from typing_extensions import (
23+
Literal as typing_extensions___Literal,
24+
)
25+
26+
27+
builtin___bool = bool
28+
builtin___bytes = bytes
29+
builtin___float = float
30+
builtin___int = int
31+
32+
33+
DESCRIPTOR: google___protobuf___descriptor___FileDescriptor = ...
34+
35+
class Duration(google___protobuf___message___Message, google___protobuf___internal___well_known_types___Duration):
36+
DESCRIPTOR: google___protobuf___descriptor___Descriptor = ...
37+
seconds: builtin___int = ...
38+
nanos: builtin___int = ...
39+
40+
def __init__(self,
41+
*,
42+
seconds : typing___Optional[builtin___int] = None,
43+
nanos : typing___Optional[builtin___int] = None,
44+
) -> None: ...
45+
def ClearField(self, field_name: typing_extensions___Literal[u"nanos",b"nanos",u"seconds",b"seconds"]) -> None: ...
46+
type___Duration = Duration

‎generated/google/protobuf/empty_pb2.pyi.expected

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
"""
2+
@generated by mypy-protobuf. Do not edit manually!
3+
isort:skip_file
4+
"""
5+
from google.protobuf.descriptor import (
6+
Descriptor as google___protobuf___descriptor___Descriptor,
7+
FileDescriptor as google___protobuf___descriptor___FileDescriptor,
8+
)
9+
10+
from google.protobuf.message import (
11+
Message as google___protobuf___message___Message,
12+
)
13+
14+
15+
DESCRIPTOR: google___protobuf___descriptor___FileDescriptor = ...
16+
17+
class Empty(google___protobuf___message___Message):
18+
DESCRIPTOR: google___protobuf___descriptor___Descriptor = ...
19+
20+
def __init__(self,
21+
) -> None: ...
22+
type___Empty = Empty

‎proto/google/protobuf/duration.proto

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
// Protocol Buffers - Google's data interchange format
2+
// Copyright 2008 Google Inc. All rights reserved.
3+
// https://developers.google.com/protocol-buffers/
4+
//
5+
// Redistribution and use in source and binary forms, with or without
6+
// modification, are permitted provided that the following conditions are
7+
// met:
8+
//
9+
// * Redistributions of source code must retain the above copyright
10+
// notice, this list of conditions and the following disclaimer.
11+
// * Redistributions in binary form must reproduce the above
12+
// copyright notice, this list of conditions and the following disclaimer
13+
// in the documentation and/or other materials provided with the
14+
// distribution.
15+
// * Neither the name of Google Inc. nor the names of its
16+
// contributors may be used to endorse or promote products derived from
17+
// this software without specific prior written permission.
18+
//
19+
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20+
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21+
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22+
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23+
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24+
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25+
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26+
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27+
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28+
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29+
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30+
31+
syntax = "proto3";
32+
33+
package google.protobuf;
34+
35+
option csharp_namespace = "Google.Protobuf.WellKnownTypes";
36+
option cc_enable_arenas = true;
37+
option go_package = "github.com/golang/protobuf/ptypes/duration";
38+
option java_package = "com.google.protobuf";
39+
option java_outer_classname = "DurationProto";
40+
option java_multiple_files = true;
41+
option objc_class_prefix = "GPB";
42+
43+
// A Duration represents a signed, fixed-length span of time represented
44+
// as a count of seconds and fractions of seconds at nanosecond
45+
// resolution. It is independent of any calendar and concepts like "day"
46+
// or "month". It is related to Timestamp in that the difference between
47+
// two Timestamp values is a Duration and it can be added or subtracted
48+
// from a Timestamp. Range is approximately +-10,000 years.
49+
//
50+
// # Examples
51+
//
52+
// Example 1: Compute Duration from two Timestamps in pseudo code.
53+
//
54+
// Timestamp start = ...;
55+
// Timestamp end = ...;
56+
// Duration duration = ...;
57+
//
58+
// duration.seconds = end.seconds - start.seconds;
59+
// duration.nanos = end.nanos - start.nanos;
60+
//
61+
// if (duration.seconds < 0 && duration.nanos > 0) {
62+
// duration.seconds += 1;
63+
// duration.nanos -= 1000000000;
64+
// } else if (duration.seconds > 0 && duration.nanos < 0) {
65+
// duration.seconds -= 1;
66+
// duration.nanos += 1000000000;
67+
// }
68+
//
69+
// Example 2: Compute Timestamp from Timestamp + Duration in pseudo code.
70+
//
71+
// Timestamp start = ...;
72+
// Duration duration = ...;
73+
// Timestamp end = ...;
74+
//
75+
// end.seconds = start.seconds + duration.seconds;
76+
// end.nanos = start.nanos + duration.nanos;
77+
//
78+
// if (end.nanos < 0) {
79+
// end.seconds -= 1;
80+
// end.nanos += 1000000000;
81+
// } else if (end.nanos >= 1000000000) {
82+
// end.seconds += 1;
83+
// end.nanos -= 1000000000;
84+
// }
85+
//
86+
// Example 3: Compute Duration from datetime.timedelta in Python.
87+
//
88+
// td = datetime.timedelta(days=3, minutes=10)
89+
// duration = Duration()
90+
// duration.FromTimedelta(td)
91+
//
92+
// # JSON Mapping
93+
//
94+
// In JSON format, the Duration type is encoded as a string rather than an
95+
// object, where the string ends in the suffix "s" (indicating seconds) and
96+
// is preceded by the number of seconds, with nanoseconds expressed as
97+
// fractional seconds. For example, 3 seconds with 0 nanoseconds should be
98+
// encoded in JSON format as "3s", while 3 seconds and 1 nanosecond should
99+
// be expressed in JSON format as "3.000000001s", and 3 seconds and 1
100+
// microsecond should be expressed in JSON format as "3.000001s".
101+
//
102+
//
103+
message Duration {
104+
// Signed seconds of the span of time. Must be from -315,576,000,000
105+
// to +315,576,000,000 inclusive. Note: these bounds are computed from:
106+
// 60 sec/min * 60 min/hr * 24 hr/day * 365.25 days/year * 10000 years
107+
int64 seconds = 1;
108+
109+
// Signed fractions of a second at nanosecond resolution of the span
110+
// of time. Durations less than one second are represented with a 0
111+
// `seconds` field and a positive or negative `nanos` field. For durations
112+
// of one second or more, a non-zero value for the `nanos` field must be
113+
// of the same sign as the `seconds` field. Must be from -999,999,999
114+
// to +999,999,999 inclusive.
115+
int32 nanos = 2;
116+
}

‎python/mypy_protobuf.py

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import google.protobuf.descriptor_pb2 as d
1212
import six
1313
from google.protobuf.compiler import plugin_pb2 as plugin_pb2
14+
from google.protobuf.internal.well_known_types import WKTBASES
1415

1516
MYPY = False
1617
if MYPY:
@@ -257,7 +258,21 @@ def write_messages(self, messages, prefix):
257258
for desc in [m for m in messages if m.name not in PYTHON_RESERVED]:
258259
self.locals.add(desc.name)
259260
qualified_name = prefix + desc.name
260-
l("class {}({}):", desc.name, message_class)
261+
262+
# Reproduce some hardcoded logic from the protobuf implementation - where
263+
# some specific "well_known_types" generated protos to have additional
264+
# base classes
265+
addl_base = u""
266+
if self.fd.package + "." + desc.name in WKTBASES:
267+
# chop off the .proto - and import the well known type
268+
# eg `from google.protobuf.duration import Duration`
269+
well_known_type = WKTBASES[self.fd.package + "." + desc.name]
270+
addl_base = ", " + self._import(
271+
"google.protobuf.internal.well_known_types",
272+
well_known_type.__name__,
273+
)
274+
275+
l("class {}({}{}):", desc.name, message_class, addl_base)
261276
with self._indent():
262277
l(
263278
"DESCRIPTOR: {} = ...",

‎run_test.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -112,5 +112,5 @@ find generated -type f -not \( -name "*.expected" -or -name "__init__.py" \) -de
112112
python --version
113113
py.test --version
114114
if [[ $PY_VER_UNIT_TESTS =~ ^2.* ]]; then IGNORE="--ignore=test/test_grpc_usage.py"; else IGNORE=""; fi
115-
PYTHONPATH=generated py.test --ignore=generated $IGNORE
115+
PYTHONPATH=generated py.test --ignore=generated $IGNORE -v
116116
)

‎test/test_generated_mypy.py

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -76,10 +76,12 @@ def compare_pyi_to_expected(output_path):
7676

7777
def test_generate_mypy_matches():
7878
# type: () -> None
79-
proto_files = glob.glob("proto/testproto/*.proto") + glob.glob(
80-
"proto/testproto/*/*.proto"
79+
proto_files = (
80+
glob.glob("proto/testproto/*.proto")
81+
+ glob.glob("proto/testproto/*/*.proto")
82+
+ glob.glob("proto/google/protobuf/*.proto")
8183
)
82-
assert len(proto_files) == 12 # Just a sanity check that all the files show up
84+
assert len(proto_files) == 13 # Just a sanity check that all the files show up
8385

8486
failures = []
8587
for fn in proto_files:

‎test_negative/negative.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,9 @@
8787
# changes to typeshed a few months after the 1.24 release
8888
# See https://github.com/python/typeshed/pull/4833
8989
_ = s.Extensions["foo"] # E:2.7 E:3.5
90+
# TODO - these will give errors once again once we are able to
91+
# revert https://github.com/python/typeshed/pull/4833 later on
92+
# a few months after 1.24 releases
9093
_ = s.Extensions[SeparateFileExtension.ext]
9194
_ = SeparateFileExtension.ext in s.Extensions
9295
del s.Extensions[SeparateFileExtension.ext]

‎test_negative/output.expected.2.7

Lines changed: 23 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -30,26 +30,26 @@ test_negative/negative.py:89: error: No overload variant of "__getitem__" of "_E
3030
test_negative/negative.py:89: note: Possible overload variants:
3131
test_negative/negative.py:89: note: def [_ExtenderMessageT <: Message] __getitem__(self, _ExtensionFieldDescriptor[Simple1, _ExtenderMessageT]) -> _ExtenderMessageT
3232
test_negative/negative.py:89: note: def __getitem__(self, FieldDescriptor) -> Any
33-
test_negative/negative.py:99: error: Incompatible types in assignment (expression has type "int", variable has type "_ExtensionFieldDescriptor[Simple1, Any]")
34-
test_negative/negative.py:103: error: Incompatible types in assignment (expression has type "Union[Literal[u'b_oneof_1'], Literal[u'b_oneof_2']]", variable has type "Union[Literal[u'a_oneof_1'], Literal[u'a_oneof_2'], Literal[u'outer_message_in_oneof'], Literal[u'outer_enum_in_oneof'], Literal[u'inner_enum_in_oneof']]")
35-
test_negative/negative.py:106: error: "Descriptor" has no attribute "Garbage"
36-
test_negative/negative.py:107: error: "Descriptor" has no attribute "Garbage"
37-
test_negative/negative.py:110: error: "EnumDescriptor" has no attribute "Garbage"
38-
test_negative/negative.py:113: error: "FileDescriptor" has no attribute "Garbage"
39-
test_negative/negative.py:120: error: "OuterEnumValue" has no attribute "FOO"
40-
test_negative/negative.py:124: error: Argument 1 to "Name" of "_EnumTypeWrapper" has incompatible type "int"; expected "OuterEnumValue"
41-
test_negative/negative.py:126: error: Argument 1 to "Name" of "_EnumTypeWrapper" has incompatible type "InnerEnumValue"; expected "OuterEnumValue"
42-
test_negative/negative.py:128: error: Argument 1 to "Name" of "_EnumTypeWrapper" has incompatible type "InnerEnumValue"; expected "OuterEnumValue"
43-
test_negative/negative.py:130: error: Argument 1 to "Name" of "_EnumTypeWrapper" has incompatible type "InnerEnumValue"; expected "OuterEnumValue"
44-
test_negative/negative.py:134: error: "ScalarMap[int, unicode]" has no attribute "get_or_create"
45-
test_negative/negative.py:136: error: No overload variant of "get" of "Mapping" matches argument type "str"
46-
test_negative/negative.py:136: note: Possible overload variant:
47-
test_negative/negative.py:136: note: def get(self, k: int) -> Optional[unicode]
48-
test_negative/negative.py:136: note: <1 more non-matching overload not shown>
49-
test_negative/negative.py:137: error: No overload variant of "get" of "Mapping" matches argument type "str"
50-
test_negative/negative.py:137: note: Possible overload variant:
51-
test_negative/negative.py:137: note: def get(self, k: int) -> Optional[OuterMessage3]
52-
test_negative/negative.py:137: note: <1 more non-matching overload not shown>
53-
test_negative/negative.py:140: error: Incompatible types in assignment (expression has type "Optional[unicode]", variable has type "int")
54-
test_negative/negative.py:141: error: Incompatible types in assignment (expression has type "Optional[OuterMessage3]", variable has type "int")
55-
Found 41 errors in 1 file (checked 16 source files)
33+
test_negative/negative.py:102: error: Incompatible types in assignment (expression has type "int", variable has type "_ExtensionFieldDescriptor[Simple1, Any]")
34+
test_negative/negative.py:106: error: Incompatible types in assignment (expression has type "Union[Literal[u'b_oneof_1'], Literal[u'b_oneof_2']]", variable has type "Union[Literal[u'a_oneof_1'], Literal[u'a_oneof_2'], Literal[u'outer_message_in_oneof'], Literal[u'outer_enum_in_oneof'], Literal[u'inner_enum_in_oneof']]")
35+
test_negative/negative.py:109: error: "Descriptor" has no attribute "Garbage"
36+
test_negative/negative.py:110: error: "Descriptor" has no attribute "Garbage"
37+
test_negative/negative.py:113: error: "EnumDescriptor" has no attribute "Garbage"
38+
test_negative/negative.py:116: error: "FileDescriptor" has no attribute "Garbage"
39+
test_negative/negative.py:123: error: "OuterEnumValue" has no attribute "FOO"
40+
test_negative/negative.py:127: error: Argument 1 to "Name" of "_EnumTypeWrapper" has incompatible type "int"; expected "OuterEnumValue"
41+
test_negative/negative.py:129: error: Argument 1 to "Name" of "_EnumTypeWrapper" has incompatible type "InnerEnumValue"; expected "OuterEnumValue"
42+
test_negative/negative.py:131: error: Argument 1 to "Name" of "_EnumTypeWrapper" has incompatible type "InnerEnumValue"; expected "OuterEnumValue"
43+
test_negative/negative.py:133: error: Argument 1 to "Name" of "_EnumTypeWrapper" has incompatible type "InnerEnumValue"; expected "OuterEnumValue"
44+
test_negative/negative.py:137: error: "ScalarMap[int, unicode]" has no attribute "get_or_create"
45+
test_negative/negative.py:139: error: No overload variant of "get" of "Mapping" matches argument type "str"
46+
test_negative/negative.py:139: note: Possible overload variant:
47+
test_negative/negative.py:139: note: def get(self, k: int) -> Optional[unicode]
48+
test_negative/negative.py:139: note: <1 more non-matching overload not shown>
49+
test_negative/negative.py:140: error: No overload variant of "get" of "Mapping" matches argument type "str"
50+
test_negative/negative.py:140: note: Possible overload variant:
51+
test_negative/negative.py:140: note: def get(self, k: int) -> Optional[OuterMessage3]
52+
test_negative/negative.py:140: note: <1 more non-matching overload not shown>
53+
test_negative/negative.py:143: error: Incompatible types in assignment (expression has type "Optional[unicode]", variable has type "int")
54+
test_negative/negative.py:144: error: Incompatible types in assignment (expression has type "Optional[OuterMessage3]", variable has type "int")
55+
Found 41 errors in 1 file (checked 17 source files)

0 commit comments

Comments
 (0)