Add support for well_known_types

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 - Support for `grpcio` stubs generation
 - Allow `mypy_protobuf.py` to be run directly as a script
+- Add support for proto's [`well_known_types`](https://developers.google.com/protocol-buffers/docs/reference/python-generated#wkt)
 
 ## 1.24
 

generated/google/protobuf/duration_pb2.pyi.expected

@@ -0,0 +1,46 @@
+"""
+@generated by mypy-protobuf.  Do not edit manually!
+isort:skip_file
+"""
+from google.protobuf.descriptor import (
+    Descriptor as google___protobuf___descriptor___Descriptor,
+    FileDescriptor as google___protobuf___descriptor___FileDescriptor,
+)
+
+from google.protobuf.internal.well_known_types import (
+    Duration as google___protobuf___internal___well_known_types___Duration,
+)
+
+from google.protobuf.message import (
+    Message as google___protobuf___message___Message,
+)
+
+from typing import (
+    Optional as typing___Optional,
+)
+
+from typing_extensions import (
+    Literal as typing_extensions___Literal,
+)
+
+
+builtin___bool = bool
+builtin___bytes = bytes
+builtin___float = float
+builtin___int = int
+
+
+DESCRIPTOR: google___protobuf___descriptor___FileDescriptor = ...
+
+class Duration(google___protobuf___message___Message, google___protobuf___internal___well_known_types___Duration):
+    DESCRIPTOR: google___protobuf___descriptor___Descriptor = ...
+    seconds: builtin___int = ...
+    nanos: builtin___int = ...
+
+    def __init__(self,
+        *,
+        seconds : typing___Optional[builtin___int] = None,
+        nanos : typing___Optional[builtin___int] = None,
+        ) -> None: ...
+    def ClearField(self, field_name: typing_extensions___Literal[u"nanos",b"nanos",u"seconds",b"seconds"]) -> None: ...
+type___Duration = Duration

generated/google/protobuf/empty_pb2.pyi.expected

@@ -0,0 +1,22 @@
+"""
+@generated by mypy-protobuf.  Do not edit manually!
+isort:skip_file
+"""
+from google.protobuf.descriptor import (
+    Descriptor as google___protobuf___descriptor___Descriptor,
+    FileDescriptor as google___protobuf___descriptor___FileDescriptor,
+)
+
+from google.protobuf.message import (
+    Message as google___protobuf___message___Message,
+)
+
+
+DESCRIPTOR: google___protobuf___descriptor___FileDescriptor = ...
+
+class Empty(google___protobuf___message___Message):
+    DESCRIPTOR: google___protobuf___descriptor___Descriptor = ...
+
+    def __init__(self,
+        ) -> None: ...
+type___Empty = Empty

proto/google/protobuf/duration.proto

@@ -0,0 +1,116 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+syntax = "proto3";
+
+package google.protobuf;
+
+option csharp_namespace = "Google.Protobuf.WellKnownTypes";
+option cc_enable_arenas = true;
+option go_package = "github.com/golang/protobuf/ptypes/duration";
+option java_package = "com.google.protobuf";
+option java_outer_classname = "DurationProto";
+option java_multiple_files = true;
+option objc_class_prefix = "GPB";
+
+// A Duration represents a signed, fixed-length span of time represented
+// as a count of seconds and fractions of seconds at nanosecond
+// resolution. It is independent of any calendar and concepts like "day"
+// or "month". It is related to Timestamp in that the difference between
+// two Timestamp values is a Duration and it can be added or subtracted
+// from a Timestamp. Range is approximately +-10,000 years.
+//
+// # Examples
+//
+// Example 1: Compute Duration from two Timestamps in pseudo code.
+//
+//     Timestamp start = ...;
+//     Timestamp end = ...;
+//     Duration duration = ...;
+//
+//     duration.seconds = end.seconds - start.seconds;
+//     duration.nanos = end.nanos - start.nanos;
+//
+//     if (duration.seconds < 0 && duration.nanos > 0) {
+//       duration.seconds += 1;
+//       duration.nanos -= 1000000000;
+//     } else if (duration.seconds > 0 && duration.nanos < 0) {
+//       duration.seconds -= 1;
+//       duration.nanos += 1000000000;
+//     }
+//
+// Example 2: Compute Timestamp from Timestamp + Duration in pseudo code.
+//
+//     Timestamp start = ...;
+//     Duration duration = ...;
+//     Timestamp end = ...;
+//
+//     end.seconds = start.seconds + duration.seconds;
+//     end.nanos = start.nanos + duration.nanos;
+//
+//     if (end.nanos < 0) {
+//       end.seconds -= 1;
+//       end.nanos += 1000000000;
+//     } else if (end.nanos >= 1000000000) {
+//       end.seconds += 1;
+//       end.nanos -= 1000000000;
+//     }
+//
+// Example 3: Compute Duration from datetime.timedelta in Python.
+//
+//     td = datetime.timedelta(days=3, minutes=10)
+//     duration = Duration()
+//     duration.FromTimedelta(td)
+//
+// # JSON Mapping
+//
+// In JSON format, the Duration type is encoded as a string rather than an
+// object, where the string ends in the suffix "s" (indicating seconds) and
+// is preceded by the number of seconds, with nanoseconds expressed as
+// fractional seconds. For example, 3 seconds with 0 nanoseconds should be
+// encoded in JSON format as "3s", while 3 seconds and 1 nanosecond should
+// be expressed in JSON format as "3.000000001s", and 3 seconds and 1
+// microsecond should be expressed in JSON format as "3.000001s".
+//
+//
+message Duration {
+  // Signed seconds of the span of time. Must be from -315,576,000,000
+  // to +315,576,000,000 inclusive. Note: these bounds are computed from:
+  // 60 sec/min * 60 min/hr * 24 hr/day * 365.25 days/year * 10000 years
+  int64 seconds = 1;
+
+  // Signed fractions of a second at nanosecond resolution of the span
+  // of time. Durations less than one second are represented with a 0
+  // `seconds` field and a positive or negative `nanos` field. For durations
+  // of one second or more, a non-zero value for the `nanos` field must be
+  // of the same sign as the `seconds` field. Must be from -999,999,999
+  // to +999,999,999 inclusive.
+  int32 nanos = 2;
+}

python/mypy_protobuf.py

@@ -11,6 +11,7 @@
 import google.protobuf.descriptor_pb2 as d
 import six
 from google.protobuf.compiler import plugin_pb2 as plugin_pb2
+from google.protobuf.internal.well_known_types import WKTBASES
 
 MYPY = False
 if MYPY:
@@ -257,7 +258,21 @@ def write_messages(self, messages, prefix):
         for desc in [m for m in messages if m.name not in PYTHON_RESERVED]:
             self.locals.add(desc.name)
             qualified_name = prefix + desc.name
-            l("class {}({}):", desc.name, message_class)
+
+            # Reproduce some hardcoded logic from the protobuf implementation - where
+            # some specific "well_known_types" generated protos to have additional
+            # base classes
+            addl_base = u""
+            if self.fd.package + "." + desc.name in WKTBASES:
+                # chop off the .proto - and import the well known type
+                # eg `from google.protobuf.duration import Duration`
+                well_known_type = WKTBASES[self.fd.package + "." + desc.name]
+                addl_base = ", " + self._import(
+                    "google.protobuf.internal.well_known_types",
+                    well_known_type.__name__,
+                )
+
+            l("class {}({}{}):", desc.name, message_class, addl_base)
             with self._indent():
                 l(
                     "DESCRIPTOR: {} = ...",

run_test.sh

@@ -112,5 +112,5 @@ find generated -type f -not \( -name "*.expected" -or -name "__init__.py" \) -de
     python --version
     py.test --version
     if [[ $PY_VER_UNIT_TESTS =~ ^2.* ]]; then IGNORE="--ignore=test/test_grpc_usage.py"; else IGNORE=""; fi
-    PYTHONPATH=generated py.test --ignore=generated $IGNORE
+    PYTHONPATH=generated py.test --ignore=generated $IGNORE -v
 )

test/test_generated_mypy.py

@@ -76,10 +76,12 @@ def compare_pyi_to_expected(output_path):
 
 def test_generate_mypy_matches():
     # type: () -> None
-    proto_files = glob.glob("proto/testproto/*.proto") + glob.glob(
-        "proto/testproto/*/*.proto"
+    proto_files = (
+        glob.glob("proto/testproto/*.proto")
+        + glob.glob("proto/testproto/*/*.proto")
+        + glob.glob("proto/google/protobuf/*.proto")
     )
-    assert len(proto_files) == 12  # Just a sanity check that all the files show up
+    assert len(proto_files) == 13  # Just a sanity check that all the files show up
 
     failures = []
     for fn in proto_files:

test_negative/negative.py

@@ -87,6 +87,9 @@
 # changes to typeshed a few months after the 1.24 release
 # See https://github.com/python/typeshed/pull/4833
 _ = s.Extensions["foo"]  # E:2.7 E:3.5
+# TODO - these will give errors once again once we are able to
+# revert https://github.com/python/typeshed/pull/4833 later on
+# a few months after 1.24 releases
 _ = s.Extensions[SeparateFileExtension.ext]
 _ = SeparateFileExtension.ext in s.Extensions
 del s.Extensions[SeparateFileExtension.ext]

test_negative/output.expected.2.7

@@ -30,26 +30,26 @@ test_negative/negative.py:89: error: No overload variant of "__getitem__" of "_E
 test_negative/negative.py:89: note: Possible overload variants:
 test_negative/negative.py:89: note:     def [_ExtenderMessageT <: Message] __getitem__(self, _ExtensionFieldDescriptor[Simple1, _ExtenderMessageT]) -> _ExtenderMessageT
 test_negative/negative.py:89: note:     def __getitem__(self, FieldDescriptor) -> Any
-test_negative/negative.py:99: error: Incompatible types in assignment (expression has type "int", variable has type "_ExtensionFieldDescriptor[Simple1, Any]")
-test_negative/negative.py:103: error: Incompatible types in assignment (expression has type "Union[Literal[u'b_oneof_1'], Literal[u'b_oneof_2']]", variable has type "Union[Literal[u'a_oneof_1'], Literal[u'a_oneof_2'], Literal[u'outer_message_in_oneof'], Literal[u'outer_enum_in_oneof'], Literal[u'inner_enum_in_oneof']]")
-test_negative/negative.py:106: error: "Descriptor" has no attribute "Garbage"
-test_negative/negative.py:107: error: "Descriptor" has no attribute "Garbage"
-test_negative/negative.py:110: error: "EnumDescriptor" has no attribute "Garbage"
-test_negative/negative.py:113: error: "FileDescriptor" has no attribute "Garbage"
-test_negative/negative.py:120: error: "OuterEnumValue" has no attribute "FOO"
-test_negative/negative.py:124: error: Argument 1 to "Name" of "_EnumTypeWrapper" has incompatible type "int"; expected "OuterEnumValue"
-test_negative/negative.py:126: error: Argument 1 to "Name" of "_EnumTypeWrapper" has incompatible type "InnerEnumValue"; expected "OuterEnumValue"
-test_negative/negative.py:128: error: Argument 1 to "Name" of "_EnumTypeWrapper" has incompatible type "InnerEnumValue"; expected "OuterEnumValue"
-test_negative/negative.py:130: error: Argument 1 to "Name" of "_EnumTypeWrapper" has incompatible type "InnerEnumValue"; expected "OuterEnumValue"
-test_negative/negative.py:134: error: "ScalarMap[int, unicode]" has no attribute "get_or_create"
-test_negative/negative.py:136: error: No overload variant of "get" of "Mapping" matches argument type "str"
-test_negative/negative.py:136: note: Possible overload variant:
-test_negative/negative.py:136: note:     def get(self, k: int) -> Optional[unicode]
-test_negative/negative.py:136: note:     <1 more non-matching overload not shown>
-test_negative/negative.py:137: error: No overload variant of "get" of "Mapping" matches argument type "str"
-test_negative/negative.py:137: note: Possible overload variant:
-test_negative/negative.py:137: note:     def get(self, k: int) -> Optional[OuterMessage3]
-test_negative/negative.py:137: note:     <1 more non-matching overload not shown>
-test_negative/negative.py:140: error: Incompatible types in assignment (expression has type "Optional[unicode]", variable has type "int")
-test_negative/negative.py:141: error: Incompatible types in assignment (expression has type "Optional[OuterMessage3]", variable has type "int")
-Found 41 errors in 1 file (checked 16 source files)
+test_negative/negative.py:102: error: Incompatible types in assignment (expression has type "int", variable has type "_ExtensionFieldDescriptor[Simple1, Any]")
+test_negative/negative.py:106: error: Incompatible types in assignment (expression has type "Union[Literal[u'b_oneof_1'], Literal[u'b_oneof_2']]", variable has type "Union[Literal[u'a_oneof_1'], Literal[u'a_oneof_2'], Literal[u'outer_message_in_oneof'], Literal[u'outer_enum_in_oneof'], Literal[u'inner_enum_in_oneof']]")
+test_negative/negative.py:109: error: "Descriptor" has no attribute "Garbage"
+test_negative/negative.py:110: error: "Descriptor" has no attribute "Garbage"
+test_negative/negative.py:113: error: "EnumDescriptor" has no attribute "Garbage"
+test_negative/negative.py:116: error: "FileDescriptor" has no attribute "Garbage"
+test_negative/negative.py:123: error: "OuterEnumValue" has no attribute "FOO"
+test_negative/negative.py:127: error: Argument 1 to "Name" of "_EnumTypeWrapper" has incompatible type "int"; expected "OuterEnumValue"
+test_negative/negative.py:129: error: Argument 1 to "Name" of "_EnumTypeWrapper" has incompatible type "InnerEnumValue"; expected "OuterEnumValue"
+test_negative/negative.py:131: error: Argument 1 to "Name" of "_EnumTypeWrapper" has incompatible type "InnerEnumValue"; expected "OuterEnumValue"
+test_negative/negative.py:133: error: Argument 1 to "Name" of "_EnumTypeWrapper" has incompatible type "InnerEnumValue"; expected "OuterEnumValue"
+test_negative/negative.py:137: error: "ScalarMap[int, unicode]" has no attribute "get_or_create"
+test_negative/negative.py:139: error: No overload variant of "get" of "Mapping" matches argument type "str"
+test_negative/negative.py:139: note: Possible overload variant:
+test_negative/negative.py:139: note:     def get(self, k: int) -> Optional[unicode]
+test_negative/negative.py:139: note:     <1 more non-matching overload not shown>
+test_negative/negative.py:140: error: No overload variant of "get" of "Mapping" matches argument type "str"
+test_negative/negative.py:140: note: Possible overload variant:
+test_negative/negative.py:140: note:     def get(self, k: int) -> Optional[OuterMessage3]
+test_negative/negative.py:140: note:     <1 more non-matching overload not shown>
+test_negative/negative.py:143: error: Incompatible types in assignment (expression has type "Optional[unicode]", variable has type "int")
+test_negative/negative.py:144: error: Incompatible types in assignment (expression has type "Optional[OuterMessage3]", variable has type "int")
+Found 41 errors in 1 file (checked 17 source files)