Skip to content

Commit 9d9b33d

Browse files
software-dovsoftware-dov
authored
feat: add fragment tests (#1056)
Fragment tests are defined by a small proto file describing an API surface with characteristics such that it is desirable to test the generated surface for correctness or to prevent regressions. As part of a fragment test, the generator is run on a fragment to create a GAPIC library for the fragment. The generated unit tests for the fragment are then executed to test the surface.
1 parent 03d8dad commit 9d9b33d

File tree

20 files changed

+1909
-217
lines changed

20 files changed

+1909
-217
lines changed

‎.github/workflows/tests.yaml‎

Lines changed: 24 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -42,39 +42,6 @@ jobs:
4242
run: python -m pip install nox
4343
- name: Check type annotations.
4444
run: nox -s mypy
45-
# publish_image:
46-
# runs-on: ubuntu-latest
47-
# container: docker
48-
# steps:
49-
# - uses: actions/checkout@v2
50-
# - setup_remote_docker
51-
# - name: Build Docker image.
52-
# run: docker build . -t gcr.io/gapic-images/gapic-generator-python:latest
53-
# - name: Download curl
54-
# run: apk add --no-cache curl
55-
# - name: Download the GCR credential helper.
56-
# run: |
57-
# curl -fsSL https://github.com/GoogleCloudPlatform/docker-credential-gcr/releases/download/v1.5.0/docker-credential-gcr_linux_amd64-1.5.0.tar.gz \
58-
# | tar xz --to-stdout ./docker-credential-gcr \
59-
# > /usr/bin/docker-credential-gcr && chmod a+x /usr/bin/docker-credential-gcr
60-
# - name: Set up authentication to Google Container Registry.
61-
# run: |
62-
# echo ${GCLOUD_SERVICE_KEY} > ${GOOGLE_APPLICATION_CREDENTIALS}
63-
# docker-credential-gcr configure-docker
64-
# - name: Tag the Docker image and push it to Google Container Registry.
65-
# run: |
66-
# if [ -n "$CIRCLE_TAG" ]; then
67-
# export MAJOR=`echo $CIRCLE_TAG | awk -F '.' '{ print $1; }'`
68-
# export MINOR=`echo $CIRCLE_TAG | awk -F '.' '{ print $2; }'`
69-
# export PATCH=`echo $CIRCLE_TAG | awk -F '.' '{ print $3; }'`
70-
# docker tag gcr.io/gapic-images/gapic-generator-python:latest gcr.io/gapic-images/gapic-generator-python:$MAJOR.$MINOR.$PATCH
71-
# docker tag gcr.io/gapic-images/gapic-generator-python:latest gcr.io/gapic-images/gapic-generator-python:$MAJOR.$MINOR
72-
# docker tag gcr.io/gapic-images/gapic-generator-python:latest gcr.io/gapic-images/gapic-generator-python:$MAJOR
73-
# docker push gcr.io/gapic-images/gapic-generator-python:$MAJOR.$MINOR.$PATCH
74-
# docker push gcr.io/gapic-images/gapic-generator-python:$MAJOR.$MINOR
75-
# docker push gcr.io/gapic-images/gapic-generator-python:$MAJOR
76-
# fi
77-
# docker push gcr.io/gapic-images/gapic-generator-python:latest
7845
showcase:
7946
strategy:
8047
matrix:
@@ -319,6 +286,30 @@ jobs:
319286
python -m pip install nox
320287
- name: Run unit tests.
321288
run: nox -s unit-${{ matrix.python }}
289+
fragment:
290+
strategy:
291+
matrix:
292+
python: [3.6, 3.7, 3.8, 3.9]
293+
runs-on: ubuntu-latest
294+
steps:
295+
- name: Cancel Previous Runs
296+
uses: styfle/[email protected]
297+
with:
298+
access_token: ${{ github.token }}
299+
- uses: actions/checkout@v2
300+
- name: Set up Python ${{ matrix.python }}
301+
uses: actions/setup-python@v2
302+
with:
303+
python-version: ${{ matrix.python }}
304+
- name: Install pandoc
305+
run: |
306+
sudo apt-get update
307+
sudo apt-get install -y pandoc gcc git
308+
- name: Install nox.
309+
run: |
310+
python -m pip install nox
311+
- name: Run fragment tests.
312+
run: nox -s fragment-${{ matrix.python }}
322313
integration:
323314
runs-on: ubuntu-latest
324315
steps:

‎WORKSPACE‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,6 @@ http_archive(
1313
url = "https://github.com/bazelbuild/rules_python/archive/0.1.0.tar.gz",
1414
)
1515

16-
load("@rules_python//python:pip.bzl", "pip_repositories")
17-
18-
pip_repositories()
19-
2016
#
2117
# Import gapic-generator-python specific dependencies
2218
#

‎gapic/templates/%namespace/%name_%version/%sub/services/%service/client.py.j2‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -405,7 +405,13 @@ class {{ service.client_name }}(metaclass={{ service.client_name }}Meta):
405405
{% endif %}
406406
{% for key, field in method.flattened_fields.items() if not field.repeated or method.input.ident.package == method.ident.package %}
407407
if {{ field.name }} is not None:
408+
{# Repeated values is a special case, because values can be lists. #}
409+
{# In order to not confuse the marshalling logic, extend these fields instead of assigning #}
410+
{% if field.ident.ident|string() == "struct_pb2.Value" and field.repeated %}
411+
request.{{ key }}.extend({{ field.name }})
412+
{% else %}
408413
request.{{ key }} = {{ field.name }}
414+
{% endif %}{# struct_pb2.Value #}
409415
{% endfor %}
410416
{# Map-y fields can be _updated_, however #}
411417
{% for key, field in method.flattened_fields.items() if field.repeated and method.input.ident.package != method.ident.package %}

‎gapic/templates/tests/unit/gapic/%name_%version/%sub/test_%service.py.j2‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -788,7 +788,15 @@ def test_{{ method.name|snake_case }}_flattened():
788788
{% elif field.ident|string() == 'duration_pb2.Duration' %}
789789
assert DurationRule().to_proto(args[0].{{ key }}) == {{ field.mock_value }}
790790
{% else %}
791-
assert args[0].{{ key }} == {{ field.mock_value }}
791+
arg = args[0].{{ key }}
792+
mock_val = {{ field.mock_value }}
793+
{% if field.ident|string() == "struct_pb2.Value" %}
794+
from proto.marshal import Marshal
795+
from proto.marshal.rules.struct import ValueRule
796+
rule = ValueRule(marshal=Marshal(name="Test"))
797+
mock_val = rule.to_python(mock_val)
798+
{% endif %}{# struct_pb2.Value #}
799+
assert arg == mock_val
792800
{% endif %}
793801
{% endif %}{% endfor %}
794802
{% for oneofs in method.flattened_oneof_fields().values() %}
@@ -873,7 +881,15 @@ async def test_{{ method.name|snake_case }}_flattened_async():
873881
{% elif field.ident|string() == 'duration_pb2.Duration' %}
874882
assert DurationRule().to_proto(args[0].{{ key }}) == {{ field.mock_value }}
875883
{% else %}
876-
assert args[0].{{ key }} == {{ field.mock_value }}
884+
arg = args[0].{{ key }}
885+
mock_val = {{ field.mock_value }}
886+
{% if field.ident|string() == "struct_pb2.Value" %}
887+
from proto.marshal import Marshal
888+
from proto.marshal.rules.struct import ValueRule
889+
rule = ValueRule(marshal=Marshal(name="Test"))
890+
mock_val = rule.to_python(mock_val)
891+
{% endif %}{# struct_pb2.Value #}
892+
assert arg == mock_val
877893
{% endif %}
878894
{% endif %}{% endfor %}
879895
{% for oneofs in method.flattened_oneof_fields().values() %}

‎noxfile.py‎

Lines changed: 113 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
# limitations under the License.
1414

1515
from __future__ import absolute_import
16+
from concurrent.futures import ThreadPoolExecutor
1617
from pathlib import Path
1718
import os
1819
import sys
@@ -29,7 +30,17 @@
2930
ADS_TEMPLATES = path.join(path.dirname(__file__), "gapic", "ads-templates")
3031

3132

32-
@nox.session(python=["3.6", "3.7", "3.8", "3.9", "3.10"])
33+
ALL_PYTHON = (
34+
"3.6",
35+
"3.7",
36+
"3.8",
37+
"3.9",
38+
)
39+
40+
NEWEST_PYTHON = "3.9"
41+
42+
43+
@nox.session(python=ALL_PYTHON)
3344
def unit(session):
3445
"""Run the unit test suite."""
3546

@@ -50,11 +61,89 @@ def unit(session):
5061
"--cov-report=term",
5162
"--cov-fail-under=100",
5263
path.join("tests", "unit"),
53-
]
64+
]
5465
),
5566
)
5667

5768

69+
FRAG_DIR = Path("tests") / "fragments"
70+
FRAGMENT_FILES = tuple(
71+
Path(dirname).relative_to(FRAG_DIR) / f
72+
for dirname, _, files in os.walk(FRAG_DIR)
73+
for f in files
74+
if os.path.splitext(f)[1] == ".proto" and f.startswith("test_")
75+
)
76+
77+
# Note: this class lives outside 'fragment'
78+
# so that, if necessary, it can be pickled for a ProcessPoolExecutor
79+
# A callable class is necessary so that the session can be closed over
80+
# instead of passed in, which simplifies the invocation via map.
81+
class FragTester:
82+
def __init__(self, session):
83+
self.session = session
84+
85+
def __call__(self, frag):
86+
with tempfile.TemporaryDirectory() as tmp_dir:
87+
# Generate the fragment GAPIC.
88+
outputs = []
89+
outputs.append(
90+
self.session.run(
91+
"python",
92+
"-m",
93+
"grpc_tools.protoc",
94+
f"--proto_path={str(FRAG_DIR)}",
95+
f"--python_gapic_out={tmp_dir}",
96+
"--python_gapic_opt=transport=grpc+rest",
97+
str(frag),
98+
external=True,
99+
silent=True,
100+
)
101+
)
102+
103+
# Install the generated fragment library.
104+
# Note: install into the tempdir to prevent issues
105+
# with running pip concurrently.
106+
self.session.install(tmp_dir, "-e", ".", "-t", tmp_dir, "-qqq")
107+
108+
# Run the fragment's generated unit tests.
109+
# Don't bother parallelizing them: we already parallelize
110+
# the fragments, and there usually aren't too many tests per fragment.
111+
outputs.append(
112+
self.session.run(
113+
"py.test",
114+
"--quiet",
115+
f"--cov-config={str(Path(tmp_dir) / '.coveragerc')}",
116+
"--cov-report=term",
117+
"--cov-fail-under=100",
118+
str(Path(tmp_dir) / "tests" / "unit"),
119+
silent=True,
120+
)
121+
)
122+
123+
return "".join(outputs)
124+
125+
126+
# TODO(dovs): ads templates
127+
@nox.session(python=ALL_PYTHON)
128+
def fragment(session):
129+
session.install(
130+
"coverage",
131+
"pytest",
132+
"pytest-cov",
133+
"pytest-xdist",
134+
"asyncmock",
135+
"pytest-asyncio",
136+
"grpcio-tools",
137+
)
138+
session.install("-e", ".")
139+
140+
with ThreadPoolExecutor() as p:
141+
all_outs = p.map(FragTester(session), FRAGMENT_FILES)
142+
143+
output = "".join(all_outs)
144+
session.log(output)
145+
146+
58147
# TODO(yon-mg): -add compute context manager that includes rest transport
59148
# -add compute unit tests
60149
# (to test against temporarily while rest transport is incomplete)
@@ -114,8 +203,7 @@ def showcase_library(
114203
f"google/showcase/v1beta1/messaging.proto",
115204
)
116205
session.run(
117-
*cmd_tup,
118-
external=True,
206+
*cmd_tup, external=True,
119207
)
120208

121209
# Install the library.
@@ -124,7 +212,7 @@ def showcase_library(
124212
yield tmp_dir
125213

126214

127-
@nox.session(python="3.9")
215+
@nox.session(python=NEWEST_PYTHON)
128216
def showcase(
129217
session,
130218
templates="DEFAULT",
@@ -136,12 +224,14 @@ def showcase(
136224
with showcase_library(session, templates=templates, other_opts=other_opts):
137225
session.install("mock", "pytest", "pytest-asyncio")
138226
session.run(
139-
"py.test", "--quiet", *(session.posargs or [path.join("tests", "system")]),
227+
"py.test",
228+
"--quiet",
229+
*(session.posargs or [path.join("tests", "system")]),
140230
env=env,
141231
)
142232

143233

144-
@nox.session(python="3.9")
234+
@nox.session(python=NEWEST_PYTHON)
145235
def showcase_mtls(
146236
session,
147237
templates="DEFAULT",
@@ -161,7 +251,7 @@ def showcase_mtls(
161251
)
162252

163253

164-
@nox.session(python="3.9")
254+
@nox.session(python=NEWEST_PYTHON)
165255
def showcase_alternative_templates(session):
166256
templates = path.join(path.dirname(__file__), "gapic", "ads-templates")
167257
showcase(
@@ -172,7 +262,7 @@ def showcase_alternative_templates(session):
172262
)
173263

174264

175-
@nox.session(python="3.9")
265+
@nox.session(python=NEWEST_PYTHON)
176266
def showcase_mtls_alternative_templates(session):
177267
templates = path.join(path.dirname(__file__), "gapic", "ads-templates")
178268
showcase_mtls(
@@ -200,12 +290,12 @@ def run_showcase_unit_tests(session, fail_under=100):
200290
"--quiet",
201291
"--cov=google",
202292
"--cov-append",
203-
f"--cov-fail-under={str(fail_under)}",
293+
f"--cov-fail-under={str(fail_under)}",
204294
*(session.posargs or [path.join("tests", "unit")]),
205295
)
206296

207297

208-
@nox.session(python=["3.6", "3.7", "3.8", "3.9"])
298+
@nox.session(python=ALL_PYTHON)
209299
def showcase_unit(
210300
session, templates="DEFAULT", other_opts: typing.Iterable[str] = (),
211301
):
@@ -233,14 +323,16 @@ def showcase_unit(
233323
run_showcase_unit_tests(session, fail_under=100)
234324

235325

236-
@nox.session(python=["3.7", "3.8", "3.9"])
326+
@nox.session(python=ALL_PYTHON[1:]) # Do not test 3.6
237327
def showcase_unit_alternative_templates(session):
238-
with showcase_library(session, templates=ADS_TEMPLATES, other_opts=("old-naming",)) as lib:
328+
with showcase_library(
329+
session, templates=ADS_TEMPLATES, other_opts=("old-naming",)
330+
) as lib:
239331
session.chdir(lib)
240332
run_showcase_unit_tests(session)
241333

242334

243-
@nox.session(python=["3.9"])
335+
@nox.session(python=NEWEST_PYTHON)
244336
def showcase_unit_add_iam_methods(session):
245337
with showcase_library(session, other_opts=("add-iam-methods",)) as lib:
246338
session.chdir(lib)
@@ -257,7 +349,7 @@ def showcase_unit_add_iam_methods(session):
257349
run_showcase_unit_tests(session, fail_under=100)
258350

259351

260-
@nox.session(python="3.9")
352+
@nox.session(python=NEWEST_PYTHON)
261353
def showcase_mypy(
262354
session, templates="DEFAULT", other_opts: typing.Iterable[str] = (),
263355
):
@@ -273,12 +365,12 @@ def showcase_mypy(
273365
session.run("mypy", "--explicit-package-bases", "google")
274366

275367

276-
@nox.session(python="3.9")
368+
@nox.session(python=NEWEST_PYTHON)
277369
def showcase_mypy_alternative_templates(session):
278370
showcase_mypy(session, templates=ADS_TEMPLATES, other_opts=("old-naming",))
279371

280372

281-
@nox.session(python="3.9")
373+
@nox.session(python=NEWEST_PYTHON)
282374
def snippetgen(session):
283375
# Clone googleapis/api-common-protos which are referenced by the snippet
284376
# protos
@@ -299,14 +391,10 @@ def snippetgen(session):
299391

300392
session.install("grpcio-tools", "mock", "pytest", "pytest-asyncio")
301393

302-
session.run(
303-
"py.test",
304-
"-vv",
305-
"tests/snippetgen"
306-
)
394+
session.run("py.test", "-vv", "tests/snippetgen")
307395

308396

309-
@nox.session(python="3.9")
397+
@nox.session(python=NEWEST_PYTHON)
310398
def docs(session):
311399
"""Build the docs."""
312400

@@ -327,15 +415,10 @@ def docs(session):
327415
)
328416

329417

330-
@nox.session(python=["3.7", "3.8", "3.9"])
418+
@nox.session(python=NEWEST_PYTHON)
331419
def mypy(session):
332420
"""Perform typecheck analysis."""
333421

334-
session.install(
335-
"mypy",
336-
"types-protobuf",
337-
"types-PyYAML",
338-
"types-dataclasses"
339-
)
422+
session.install("mypy", "types-protobuf", "types-PyYAML", "types-dataclasses")
340423
session.install(".")
341424
session.run("mypy", "gapic")

0 commit comments

Comments
 (0)