fix: minor typo in ads template (#664)

CircleCI machinery now invokes the alternative (Ads) templates for the
showcase_alternative_templates_* tests.

Includes numerous fixes and additions to the Ads grpc transport,
client class, and unit test templates.

The showcase system tests now selectively enable async tests via an
environment variable. The async client code has not yet been added to
the Ads templates, and the corresponding system tests have been
disabled for alternative templates.

Author: software-dov

packages/gapic-generator/gapic/ads-templates/%namespace/%name/%version/%sub/services/%service/transports/base.py.j2

@@ -55,10 +55,10 @@ class {{ service.name }}Transport(metaclass=abc.ABCMeta):
                 credentials identify the application to the service; if none
                 are specified, the client will attempt to ascertain the
                 credentials from the environment.
-            client_info (google.api_core.gapic_v1.client_info.ClientInfo):	
-                The client info used to send a user-agent string along with	
-                API requests. If ``None``, then default info will be used.	
-                Generally, you only need to set this if you're developing	
+            client_info (google.api_core.gapic_v1.client_info.ClientInfo):
+                The client info used to send a user-agent string along with
+                API requests. If ``None``, then default info will be used.
+                Generally, you only need to set this if you're developing
                 your own client library.
         """
         # Save the hostname. Default to port 443 (HTTPS) if none is specified.
@@ -89,7 +89,7 @@ class {{ service.name }}Transport(metaclass=abc.ABCMeta):
                     {% if method.retry.max_backoff %}maximum={{ method.retry.max_backoff }},{% endif %}
                     {% if method.retry.backoff_multiplier %}multiplier={{ method.retry.backoff_multiplier }},{% endif %}
                     predicate=retries.if_exception_type(
-                        {%- for ex in method.retry.retryable_exceptions|sort(attribute='__name__) %}
+                        {%- for ex in method.retry.retryable_exceptions|sort(attribute='__name__') %}
                         exceptions.{{ ex.__name__ }},
                         {%- endfor %}
                     ),

packages/gapic-generator/gapic/ads-templates/%namespace/%name/%version/%sub/services/%service/transports/grpc.py.j2

@@ -1,7 +1,8 @@
 {% extends '_base.py.j2' %}
 
 {% block content %}
-from typing import Callable, Dict, Tuple
+import warnings
+from typing import Callable, Dict, Optional, Sequence, Tuple
 
 from google.api_core import grpc_helpers   # type: ignore
 {%- if service.has_lro %}
@@ -10,7 +11,7 @@ from google.api_core import operations_v1  # type: ignore
 from google.api_core import gapic_v1       # type: ignore
 from google import auth                    # type: ignore
 from google.auth import credentials        # type: ignore
-
+from google.auth.transport.grpc import SslCredentials  # type: ignore
 
 import grpc  # type: ignore
 
@@ -38,8 +39,13 @@ class {{ service.name }}GrpcTransport({{ service.name }}Transport):
     def __init__(self, *,
             host: str{% if service.host %} = '{{ service.host }}'{% endif %},
             credentials: credentials.Credentials = None,
+            credentials_file: str = None,
+            scopes: Sequence[str] = None,
             channel: grpc.Channel = None,
+            api_mtls_endpoint: str = None,
+            client_cert_source: Callable[[], Tuple[bytes, bytes]] = None,
             ssl_channel_credentials: grpc.ChannelCredentials = None,
+            quota_project_id: Optional[str] = None,
             client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO,
             ) -> None:
         """Instantiate the transport.
@@ -53,14 +59,29 @@ class {{ service.name }}GrpcTransport({{ service.name }}Transport):
                 are specified, the client will attempt to ascertain the
                 credentials from the environment.
                 This argument is ignored if ``channel`` is provided.
+            credentials_file (Optional[str]): A file with credentials that can
+                be loaded with :func:`google.auth.load_credentials_from_file`.
+                This argument is ignored if ``channel`` is provided.
+            scopes (Optional(Sequence[str])): A list of scopes. This argument is
+                ignored if ``channel`` is provided.
             channel (Optional[grpc.Channel]): A ``Channel`` instance through
                 which to make calls.
+            api_mtls_endpoint (Optional[str]): Deprecated. The mutual TLS endpoint.
+                If provided, it overrides the ``host`` argument and tries to create
+                a mutual TLS channel with client SSL credentials from
+                ``client_cert_source`` or applicatin default SSL credentials.
+            client_cert_source (Optional[Callable[[], Tuple[bytes, bytes]]]):
+                Deprecated. A callback to provide client SSL certificate bytes and
+                private key bytes, both in PEM format. It is ignored if
+                ``api_mtls_endpoint`` is None.
             ssl_channel_credentials (grpc.ChannelCredentials): SSL credentials
                 for grpc channel. It is ignored if ``channel`` is provided.
-            client_info (google.api_core.gapic_v1.client_info.ClientInfo):	
-                The client info used to send a user-agent string along with	
-                API requests. If ``None``, then default info will be used.	
-                Generally, you only need to set this if you're developing	
+            quota_project_id (Optional[str]): An optional project to use for billing
+                and quota.
+            client_info (google.api_core.gapic_v1.client_info.ClientInfo):
+                The client info used to send a user-agent string along with
+                API requests. If ``None``, then default info will be used.
+                Generally, you only need to set this if you're developing
                 your own client library.
 
         Raises:
@@ -74,14 +95,41 @@ class {{ service.name }}GrpcTransport({{ service.name }}Transport):
 
             # If a channel was explicitly provided, set it.
             self._grpc_channel = channel
+        elif api_mtls_endpoint:
+            warnings.warn("api_mtls_endpoint and client_cert_source are deprecated", DeprecationWarning)
+
+            host = api_mtls_endpoint if ":" in api_mtls_endpoint else api_mtls_endpoint + ":443"
+
+            if credentials is None:
+                credentials, _ = auth.default(scopes=self.AUTH_SCOPES, quota_project_id=quota_project_id)
+
+            # Create SSL credentials with client_cert_source or application
+            # default SSL credentials.
+            if client_cert_source:
+                cert, key = client_cert_source()
+                ssl_credentials = grpc.ssl_channel_credentials(
+                    certificate_chain=cert, private_key=key
+                )
+            else:
+                ssl_credentials = SslCredentials().ssl_credentials
+
+            # create a new channel. The provided one is ignored.
+            self._grpc_channel = type(self).create_channel(
+                host,
+                credentials=credentials,
+                credentials_file=credentials_file,
+                ssl_credentials=ssl_credentials,
+                scopes=scopes or self.AUTH_SCOPES,
+                quota_project_id=quota_project_id,
+            )
         else:
             host = host if ":" in host else host + ":443"
 
             if credentials is None:
                 credentials, _ = auth.default(scopes=self.AUTH_SCOPES)
 
             # create a new channel. The provided one is ignored.
-            self._grpc_channel = grpc_helpers.create_channel(
+            self._grpc_channel = type(self).create_channel(
                 host,
                 credentials=credentials,
                 ssl_credentials=ssl_channel_credentials,
@@ -102,6 +150,7 @@ class {{ service.name }}GrpcTransport({{ service.name }}Transport):
     def create_channel(cls,
                        host: str{% if service.host %} = '{{ service.host }}'{% endif %},
                        credentials: credentials.Credentials = None,
+                       scopes: Optional[Sequence[str]] = None,
                        **kwargs) -> grpc.Channel:
         """Create and return a gRPC channel object.
         Args:
@@ -111,6 +160,9 @@ class {{ service.name }}GrpcTransport({{ service.name }}Transport):
                 credentials identify this application to the service. If
                 none are specified, the client will attempt to ascertain
                 the credentials from the environment.
+            scopes (Optional[Sequence[str]]): A optional list of scopes needed for this
+                service. These are only used when credentials are not specified and
+                are passed to :func:`google.auth.default`.
             kwargs (Optional[dict]): Keyword arguments, which are passed to the
                 channel creation.
         Returns:
@@ -119,26 +171,14 @@ class {{ service.name }}GrpcTransport({{ service.name }}Transport):
         return grpc_helpers.create_channel(
             host,
             credentials=credentials,
-            scopes=cls.AUTH_SCOPES,
+            scopes=scopes or cls.AUTH_SCOPES,
             **kwargs
         )
 
     @property
     def grpc_channel(self) -> grpc.Channel:
-        """Create the channel designed to connect to this service.
-
-        This property caches on the instance; repeated calls return
-        the same channel.
+        """Return the channel designed to connect to this service.
         """
-        # Sanity check: Only create a new channel if we do not already
-        # have one.
-        if not hasattr(self, '_grpc_channel'):
-            self._grpc_channel = self.create_channel(
-                self._host,
-                credentials=self._credentials,
-            )
-
-        # Return the channel from cache.
         return self._grpc_channel
     {%- if service.has_lro %}
 

packages/gapic-generator/gapic/ads-templates/noxfile.py.j2

@@ -20,7 +20,7 @@ def unit(session):
         '--cov-config=.coveragerc',
         '--cov-report=term',
         '--cov-report=html',
-        os.path.join('tests', 'unit', '{{ api.naming.versioned_module_name }}'),
+        os.path.join('tests', 'unit', 'gapic', '{{ api.naming.versioned_module_name }}'),
     )
 
 

packages/gapic-generator/gapic/ads-templates/setup.py.j2

@@ -24,7 +24,7 @@ setuptools.setup(
         'grpc-google-iam-v1',
     {%- endif %}
     ),
-    python_requires='>={% if opts.lazy_import %}3.7{% else %}3.6{% endif %}',{# Lazy import requires module-level getattr #}
+    python_requires='>=3.7',{# Lazy import requires module-level getattr #}
     setup_requires=[
         'libcst >= 0.2.5',
     ],

packages/gapic-generator/gapic/ads-templates/tests/unit/gapic/%name_%version/%sub/test_%service.py.j2

@@ -144,7 +144,7 @@ def test_{{ service.client_name|snake_case }}_client_options():
     # Check the case GOOGLE_API_USE_CLIENT_CERTIFICATE has unsupported value.
     with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"}):
         with pytest.raises(ValueError):
-            client = client_class()
+            client = {{ service.client_name }}()
 
 
 @mock.patch.object({{ service.client_name }}, "DEFAULT_ENDPOINT", modify_default_endpoint({{ service.client_name }}))
@@ -222,7 +222,7 @@ def test_{{ service.client_name|snake_case }}_mtls_env_auto(use_client_cert_env)
 
 
 def test_{{ service.client_name|snake_case }}_client_options_from_dict():
-    with mock.patch('{{ (api.naming.module_namespace + (api.naming.versioned_module_name,) + service.meta.address.subpackage)|join(".") }}.services.{{ service.name|snake_case }}.transports.{{ service.name }}Transport.__init__') as grpc_transport:
+    with mock.patch('{{ (api.naming.module_namespace + (api.naming.versioned_module_name,) + service.meta.address.subpackage)|join(".") }}.services.{{ service.name|snake_case }}.transports.{{ service.name }}GrpcTransport.__init__') as grpc_transport:
         grpc_transport.return_value = None
         client = {{ service.client_name }}(
             client_options={'api_endpoint': 'squid.clam.whelk'}
@@ -583,6 +583,15 @@ def test_transport_instance():
     assert client.transport is transport
 
 
+def test_transport_get_channel():
+    # A client may be instantiated with a custom transport instance.
+    transport = transports.{{ service.name }}GrpcTransport(
+        credentials=credentials.AnonymousCredentials(),
+    )
+    channel = transport.grpc_channel
+    assert channel
+
+
 def test_transport_grpc_default():
     # A client should use the gRPC transport by default.
     client = {{ service.client_name }}(
@@ -593,18 +602,20 @@ def test_transport_grpc_default():
         transports.{{ service.name }}GrpcTransport,
     )
 
-
-def test_transport_adc():
+@pytest.mark.parametrize("transport_class", [
+    transports.{{ service.grpc_transport_name }},
+])
+def test_transport_adc(transport_class):
     # Test default credentials are used if not provided.
     with mock.patch.object(auth, 'default') as adc:
         adc.return_value = (credentials.AnonymousCredentials(), None)
-        transports.{{ service.name }}Transport()
+        transport_class()
         adc.assert_called_once()
 
 
 def test_{{ service.name|snake_case }}_base_transport():
     # Instantiate the base transport.
-    with mock.patch('{{ (api.naming.module_namespace + (api.naming.versioned_module_name,) + service.meta.address.subpackage)|join(".") }}.services.{{ service.name|snake_case }}.transports.{{ service.name }}GrpcTransport.__init__') as Transport:
+    with mock.patch('{{ (api.naming.module_namespace + (api.naming.versioned_module_name,) + service.meta.address.subpackage)|join(".") }}.services.{{ service.name|snake_case }}.transports.{{ service.name }}Transport.__init__') as Transport:
         Transport.return_value = None
         transport = transports.{{ service.name }}Transport(
             credentials=credentials.AnonymousCredentials(),
@@ -695,6 +706,85 @@ def test_{{ service.name|snake_case }}_grpc_transport_channel():
     assert transport._host == "squid.clam.whelk:443"
 
 
+@pytest.mark.parametrize("transport_class", [transports.{{ service.grpc_transport_name }}])
+def test_{{ service.name|snake_case }}_transport_channel_mtls_with_client_cert_source(
+    transport_class
+):
+    with mock.patch("grpc.ssl_channel_credentials", autospec=True) as grpc_ssl_channel_cred:
+        with mock.patch.object(transport_class, "create_channel", autospec=True) as grpc_create_channel:
+            mock_ssl_cred = mock.Mock()
+            grpc_ssl_channel_cred.return_value = mock_ssl_cred
+
+            mock_grpc_channel = mock.Mock()
+            grpc_create_channel.return_value = mock_grpc_channel
+
+            cred = credentials.AnonymousCredentials()
+            with pytest.warns(DeprecationWarning):
+                with mock.patch.object(auth, 'default') as adc:
+                    adc.return_value = (cred, None)
+                    transport = transport_class(
+                        host="squid.clam.whelk",
+                        api_mtls_endpoint="mtls.squid.clam.whelk",
+                        client_cert_source=client_cert_source_callback,
+                    )
+                    adc.assert_called_once()
+
+            grpc_ssl_channel_cred.assert_called_once_with(
+                certificate_chain=b"cert bytes", private_key=b"key bytes"
+            )
+            grpc_create_channel.assert_called_once_with(
+                "mtls.squid.clam.whelk:443",
+                credentials=cred,
+                credentials_file=None,
+                scopes=(
+                    {%- for scope in service.oauth_scopes %}
+                    '{{ scope }}',
+                    {%- endfor %}
+                ),
+                ssl_credentials=mock_ssl_cred,
+                quota_project_id=None,
+            )
+            assert transport.grpc_channel == mock_grpc_channel
+
+
+@pytest.mark.parametrize("transport_class", [transports.{{ service.grpc_transport_name }},])
+def test_{{ service.name|snake_case }}_transport_channel_mtls_with_adc(
+    transport_class
+):
+    mock_ssl_cred = mock.Mock()
+    with mock.patch.multiple(
+        "google.auth.transport.grpc.SslCredentials",
+        __init__=mock.Mock(return_value=None),
+        ssl_credentials=mock.PropertyMock(return_value=mock_ssl_cred),
+    ):
+        with mock.patch.object(transport_class, "create_channel", autospec=True) as grpc_create_channel:
+            mock_grpc_channel = mock.Mock()
+            grpc_create_channel.return_value = mock_grpc_channel
+            mock_cred = mock.Mock()
+
+            with pytest.warns(DeprecationWarning):
+                transport = transport_class(
+                    host="squid.clam.whelk",
+                    credentials=mock_cred,
+                    api_mtls_endpoint="mtls.squid.clam.whelk",
+                    client_cert_source=None,
+                )
+
+            grpc_create_channel.assert_called_once_with(
+                "mtls.squid.clam.whelk:443",
+                credentials=mock_cred,
+                credentials_file=None,
+                scopes=(
+                    {%- for scope in service.oauth_scopes %}
+                    '{{ scope }}',
+                    {%- endfor %}
+                ),
+                ssl_credentials=mock_ssl_cred,
+                quota_project_id=None,
+            )
+            assert transport.grpc_channel == mock_grpc_channel
+
+
 {% if service.has_lro -%}
 def test_{{ service.name|snake_case }}_grpc_lro_client():
     client = {{ service.client_name }}(

packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/async_client.py.j2

@@ -50,6 +50,16 @@ class {{ service.async_client_name }}:
     from_service_account_file = {{ service.client_name }}.from_service_account_file
     from_service_account_json = from_service_account_file
 
+    @property
+    def transport(self) -> {{ service.name }}Transport:
+        """Return the transport used by the client instance.
+
+        Returns:
+            {{ service.name }}Transport: The transport used by the client instance.
+        """
+        return self._client.transport
+
+
     get_transport_class = functools.partial(type({{ service.client_name }}).get_transport_class, type({{ service.client_name }}))
 
     def __init__(self, *,