Skip to content

Commit 9af49c0

Browse files
nipunn1313nipunn1313
authored
Protobuf stubs update using mypy-protobuf (#4785)
* Add script to generate protoc stubs using mypy-protobuf generated stubs * Use generate_proto_stubs to generate stubs for protobuf 3.14.0 * Skip _pb2.pyi from flake8,black,isort,pytype
1 parent 04bfaf5 commit 9af49c0

34 files changed

+2738
-4385
lines changed

‎.flake8

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,5 +22,5 @@ per-file-ignores =
2222

2323
# We are checking with Python 3 but many of the stubs are Python 2 stubs.
2424
builtins = StandardError,apply,basestring,buffer,cmp,coerce,execfile,file,intern,long,raw_input,reduce,reload,unichr,unicode,xrange
25-
exclude = .venv*,@*,.git
25+
exclude = .venv*,@*,.git,*_pb2.pyi
2626
max-line-length = 130

‎pyproject.toml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,13 @@
11
[tool.black]
22
line_length = 130
33
target_version = ["py37"]
4+
exclude = ".*_pb2.pyi"
45

56
[tool.isort]
67
profile = "black"
78
combine_as_imports = true
89
line_length = 130
10+
skip_glob = "*_pb2.pyi"
911
extra_standard_library = [
1012
"typing_extensions",
1113
"_typeshed",

‎scripts/generate_proto_stubs.sh

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
#!/bin/bash
2+
# Some of the proto .pyi stubs in third_party/2and3/google/protobuf/
3+
# are autogenerated using the mypy-protobuf project on the
4+
# latest `.proto` files shipped with protoc.
5+
#
6+
# When run, this script will autogenerate the _pb2.pyi stubs to
7+
# third_party/2and3/google/protobuf. It should be run any time there's
8+
# a meaningful update to either PROTOBUF_VERSION or MYPY_PROTOBUF_VERSION,
9+
# followed by committing the changes to typeshed
10+
#
11+
# Update these two variables when rerunning script
12+
PROTOBUF_VERSION=3.14.0
13+
MYPY_PROTOBUF_VERSION=v1.23
14+
15+
set -ex
16+
17+
if uname -a | grep Darwin; then
18+
PLAT=osx
19+
else
20+
PLAT=linux
21+
fi
22+
REPO_ROOT=$(realpath $(dirname "${BASH_SOURCE[0]}")/..)
23+
TMP_DIR=$(mktemp -d)
24+
PYTHON_PROTOBUF_FILENAME=protobuf-python-${PROTOBUF_VERSION}.zip
25+
PROTOC_FILENAME=protoc-${PROTOBUF_VERSION}-${PLAT}-x86_64.zip
26+
PROTOC_URL=https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION}/${PROTOC_FILENAME}
27+
PYTHON_PROTOBUF_URL=https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION}/${PYTHON_PROTOBUF_FILENAME}
28+
29+
cd $TMP_DIR
30+
echo "Working in $TMP_DIR"
31+
32+
# Install protoc
33+
wget $PROTOC_URL
34+
mkdir protoc_install
35+
unzip $PROTOC_FILENAME -d protoc_install
36+
37+
# Fetch protoc-python (which contains all the .proto files)
38+
wget $PYTHON_PROTOBUF_URL
39+
unzip $PYTHON_PROTOBUF_FILENAME
40+
PYTHON_PROTOBUF_DIR=protobuf-$PROTOBUF_VERSION
41+
42+
# Install mypy-protobuf
43+
VENV=venv
44+
python3 -m virtualenv $VENV
45+
source $VENV/bin/activate
46+
python3 -m pip install git+https://github.com/dropbox/mypy-protobuf@${MYPY_PROTOBUF_VERSION}#subdirectory=python
47+
48+
# Remove existing pyi
49+
find $REPO_ROOT/third_party/2and3/ -name "*_pb2.pyi" -delete
50+
51+
# Roughly reproduce the subset of .proto files on the public interface as described
52+
# by find_package_modules in the protobuf setup.py.
53+
# The logic (as of 3.14.0) can roughly be described as a whitelist of .proto files
54+
# further limited to exclude *test* and internal/
55+
# https://github.com/protocolbuffers/protobuf/blob/master/python/setup.py
56+
PROTO_FILES=$(grep "generate_proto.*google" $PYTHON_PROTOBUF_DIR/python/setup.py | \
57+
cut -d\" -f2 | \
58+
grep -v "test" | \
59+
grep -v google/protobuf/internal/ | \
60+
grep -v google/protobuf/pyext/python.proto | \
61+
sed "s:^:$PYTHON_PROTOBUF_DIR/python/:" | \
62+
xargs -L1 realpath --relative-to=. \
63+
)
64+
65+
# And regenerate!
66+
protoc_install/bin/protoc --proto_path=$PYTHON_PROTOBUF_DIR/src --mypy_out=$REPO_ROOT/third_party/2and3 $PROTO_FILES

‎tests/pytype_exclude_list.txt

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,3 +16,31 @@ third_party/2and3/attr/converters.pyi
1616
third_party/2and3/attr/filters.pyi
1717
third_party/2and3/attr/validators.pyi
1818
third_party/2and3/pynamodb/models.pyi
19+
20+
# _pb2.pyi have some constructs that break pytype
21+
# Eg
22+
# pytype.pyi.parser.ParseError: File: "/Users/nipunn/src/typeshed/third_party/2and3/google/protobuf/descriptor_pb2.pyi", line 195
23+
# b"TypeValue = typing___NewType('TypeValue', builtin___int)"
24+
third_party/2and3/google/protobuf/any_pb2.pyi
25+
third_party/2and3/google/protobuf/api_pb2.pyi
26+
third_party/2and3/google/protobuf/compiler/plugin_pb2.pyi
27+
third_party/2and3/google/protobuf/descriptor.pyi
28+
third_party/2and3/google/protobuf/descriptor_pb2.pyi
29+
third_party/2and3/google/protobuf/duration_pb2.pyi
30+
third_party/2and3/google/protobuf/empty_pb2.pyi
31+
third_party/2and3/google/protobuf/field_mask_pb2.pyi
32+
third_party/2and3/google/protobuf/internal/containers.pyi
33+
third_party/2and3/google/protobuf/internal/enum_type_wrapper.pyi
34+
third_party/2and3/google/protobuf/internal/extension_dict.pyi
35+
third_party/2and3/google/protobuf/json_format.pyi
36+
third_party/2and3/google/protobuf/message.pyi
37+
third_party/2and3/google/protobuf/message_factory.pyi
38+
third_party/2and3/google/protobuf/service.pyi
39+
third_party/2and3/google/protobuf/source_context_pb2.pyi
40+
third_party/2and3/google/protobuf/struct_pb2.pyi
41+
third_party/2and3/google/protobuf/symbol_database.pyi
42+
third_party/2and3/google/protobuf/timestamp_pb2.pyi
43+
third_party/2and3/google/protobuf/type_pb2.pyi
44+
third_party/2and3/google/protobuf/util/json_format_pb2.pyi
45+
third_party/2and3/google/protobuf/util/json_format_proto3_pb2.pyi
46+
third_party/2and3/google/protobuf/wrappers_pb2.pyi

‎third_party/2and3/google/protobuf/any_pb2.pyi

Lines changed: 39 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,41 @@
1-
from typing import Optional, Text
1+
# @generated by generate_proto_mypy_stubs.py. Do not edit!
2+
import sys
3+
from google.protobuf.descriptor import (
4+
Descriptor as google___protobuf___descriptor___Descriptor,
5+
FileDescriptor as google___protobuf___descriptor___FileDescriptor,
6+
)
27

3-
from google.protobuf.internal import well_known_types
4-
from google.protobuf.message import Message
8+
from google.protobuf.message import (
9+
Message as google___protobuf___message___Message,
10+
)
511

6-
class Any(Message, well_known_types.Any_):
7-
type_url: Text
8-
value: bytes
9-
def __init__(self, type_url: Optional[Text] = ..., value: Optional[bytes] = ...) -> None: ...
12+
from typing import (
13+
Optional as typing___Optional,
14+
Text as typing___Text,
15+
)
16+
17+
from typing_extensions import (
18+
Literal as typing_extensions___Literal,
19+
)
20+
21+
22+
builtin___bool = bool
23+
builtin___bytes = bytes
24+
builtin___float = float
25+
builtin___int = int
26+
27+
28+
DESCRIPTOR: google___protobuf___descriptor___FileDescriptor = ...
29+
30+
class Any(google___protobuf___message___Message):
31+
DESCRIPTOR: google___protobuf___descriptor___Descriptor = ...
32+
type_url: typing___Text = ...
33+
value: builtin___bytes = ...
34+
35+
def __init__(self,
36+
*,
37+
type_url : typing___Optional[typing___Text] = None,
38+
value : typing___Optional[builtin___bytes] = None,
39+
) -> None: ...
40+
def ClearField(self, field_name: typing_extensions___Literal[u"type_url",b"type_url",u"value",b"value"]) -> None: ...
41+
type___Any = Any

‎third_party/2and3/google/protobuf/any_test_pb2.pyi

Lines changed: 0 additions & 15 deletions
This file was deleted.

‎third_party/2and3/google/protobuf/api_pb2.pyi

Lines changed: 105 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -1,54 +1,112 @@
1-
from typing import Iterable, Optional, Text
1+
# @generated by generate_proto_mypy_stubs.py. Do not edit!
2+
import sys
3+
from google.protobuf.descriptor import (
4+
Descriptor as google___protobuf___descriptor___Descriptor,
5+
FileDescriptor as google___protobuf___descriptor___FileDescriptor,
6+
)
27

3-
from google.protobuf.internal.containers import RepeatedCompositeFieldContainer
4-
from google.protobuf.message import Message
5-
from google.protobuf.source_context_pb2 import SourceContext
6-
from google.protobuf.type_pb2 import Option, Syntax
8+
from google.protobuf.internal.containers import (
9+
RepeatedCompositeFieldContainer as google___protobuf___internal___containers___RepeatedCompositeFieldContainer,
10+
)
11+
12+
from google.protobuf.message import (
13+
Message as google___protobuf___message___Message,
14+
)
15+
16+
from google.protobuf.source_context_pb2 import (
17+
SourceContext as google___protobuf___source_context_pb2___SourceContext,
18+
)
19+
20+
from google.protobuf.type_pb2 import (
21+
Option as google___protobuf___type_pb2___Option,
22+
SyntaxValue as google___protobuf___type_pb2___SyntaxValue,
23+
)
24+
25+
from typing import (
26+
Iterable as typing___Iterable,
27+
Optional as typing___Optional,
28+
Text as typing___Text,
29+
)
30+
31+
from typing_extensions import (
32+
Literal as typing_extensions___Literal,
33+
)
34+
35+
36+
builtin___bool = bool
37+
builtin___bytes = bytes
38+
builtin___float = float
39+
builtin___int = int
40+
41+
42+
DESCRIPTOR: google___protobuf___descriptor___FileDescriptor = ...
43+
44+
class Api(google___protobuf___message___Message):
45+
DESCRIPTOR: google___protobuf___descriptor___Descriptor = ...
46+
name: typing___Text = ...
47+
version: typing___Text = ...
48+
syntax: google___protobuf___type_pb2___SyntaxValue = ...
749

8-
class Api(Message):
9-
name: Text
10-
version: Text
11-
syntax: Syntax
1250
@property
13-
def methods(self) -> RepeatedCompositeFieldContainer[Method]: ...
51+
def methods(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[type___Method]: ...
52+
1453
@property
15-
def options(self) -> RepeatedCompositeFieldContainer[Option]: ...
54+
def options(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[google___protobuf___type_pb2___Option]: ...
55+
1656
@property
17-
def source_context(self) -> SourceContext: ...
57+
def source_context(self) -> google___protobuf___source_context_pb2___SourceContext: ...
58+
1859
@property
19-
def mixins(self) -> RepeatedCompositeFieldContainer[Mixin]: ...
20-
def __init__(
21-
self,
22-
name: Optional[Text] = ...,
23-
methods: Optional[Iterable[Method]] = ...,
24-
options: Optional[Iterable[Option]] = ...,
25-
version: Optional[Text] = ...,
26-
source_context: Optional[SourceContext] = ...,
27-
mixins: Optional[Iterable[Mixin]] = ...,
28-
syntax: Optional[Syntax] = ...,
29-
) -> None: ...
30-
31-
class Method(Message):
32-
name: Text
33-
request_type_url: Text
34-
request_streaming: bool
35-
response_type_url: Text
36-
response_streaming: bool
37-
syntax: Syntax
60+
def mixins(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[type___Mixin]: ...
61+
62+
def __init__(self,
63+
*,
64+
name : typing___Optional[typing___Text] = None,
65+
methods : typing___Optional[typing___Iterable[type___Method]] = None,
66+
options : typing___Optional[typing___Iterable[google___protobuf___type_pb2___Option]] = None,
67+
version : typing___Optional[typing___Text] = None,
68+
source_context : typing___Optional[google___protobuf___source_context_pb2___SourceContext] = None,
69+
mixins : typing___Optional[typing___Iterable[type___Mixin]] = None,
70+
syntax : typing___Optional[google___protobuf___type_pb2___SyntaxValue] = None,
71+
) -> None: ...
72+
def HasField(self, field_name: typing_extensions___Literal[u"source_context",b"source_context"]) -> builtin___bool: ...
73+
def ClearField(self, field_name: typing_extensions___Literal[u"methods",b"methods",u"mixins",b"mixins",u"name",b"name",u"options",b"options",u"source_context",b"source_context",u"syntax",b"syntax",u"version",b"version"]) -> None: ...
74+
type___Api = Api
75+
76+
class Method(google___protobuf___message___Message):
77+
DESCRIPTOR: google___protobuf___descriptor___Descriptor = ...
78+
name: typing___Text = ...
79+
request_type_url: typing___Text = ...
80+
request_streaming: builtin___bool = ...
81+
response_type_url: typing___Text = ...
82+
response_streaming: builtin___bool = ...
83+
syntax: google___protobuf___type_pb2___SyntaxValue = ...
84+
3885
@property
39-
def options(self) -> RepeatedCompositeFieldContainer[Option]: ...
40-
def __init__(
41-
self,
42-
name: Optional[Text] = ...,
43-
request_type_url: Optional[Text] = ...,
44-
request_streaming: Optional[bool] = ...,
45-
response_type_url: Optional[Text] = ...,
46-
response_streaming: Optional[bool] = ...,
47-
options: Optional[Iterable[Option]] = ...,
48-
syntax: Optional[Syntax] = ...,
49-
) -> None: ...
50-
51-
class Mixin(Message):
52-
name: Text
53-
root: Text
54-
def __init__(self, name: Optional[Text] = ..., root: Optional[Text] = ...) -> None: ...
86+
def options(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[google___protobuf___type_pb2___Option]: ...
87+
88+
def __init__(self,
89+
*,
90+
name : typing___Optional[typing___Text] = None,
91+
request_type_url : typing___Optional[typing___Text] = None,
92+
request_streaming : typing___Optional[builtin___bool] = None,
93+
response_type_url : typing___Optional[typing___Text] = None,
94+
response_streaming : typing___Optional[builtin___bool] = None,
95+
options : typing___Optional[typing___Iterable[google___protobuf___type_pb2___Option]] = None,
96+
syntax : typing___Optional[google___protobuf___type_pb2___SyntaxValue] = None,
97+
) -> None: ...
98+
def ClearField(self, field_name: typing_extensions___Literal[u"name",b"name",u"options",b"options",u"request_streaming",b"request_streaming",u"request_type_url",b"request_type_url",u"response_streaming",b"response_streaming",u"response_type_url",b"response_type_url",u"syntax",b"syntax"]) -> None: ...
99+
type___Method = Method
100+
101+
class Mixin(google___protobuf___message___Message):
102+
DESCRIPTOR: google___protobuf___descriptor___Descriptor = ...
103+
name: typing___Text = ...
104+
root: typing___Text = ...
105+
106+
def __init__(self,
107+
*,
108+
name : typing___Optional[typing___Text] = None,
109+
root : typing___Optional[typing___Text] = None,
110+
) -> None: ...
111+
def ClearField(self, field_name: typing_extensions___Literal[u"name",b"name",u"root",b"root"]) -> None: ...
112+
type___Mixin = Mixin

0 commit comments

Comments
 (0)