diff --git a/appveyor.yml b/appveyor.yml index fbb5e382a6f2..d1cd70e73bbb 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -30,12 +30,10 @@ environment: - PYTHON: "C:\\Python34" PYTHON_VERSION: "3.4.4" PYTHON_ARCH: "32" - # Use mocked-up GRPC for now - PYTHON: "C:\\Python34-x64" PYTHON_VERSION: "3.4.4" PYTHON_ARCH: "64" - # Use mocked-up GRPC for now # Python 3.5.1 is the latest Python 3.5 with a Windows installer # Python 3.5.1 is the overall latest @@ -43,12 +41,10 @@ environment: - PYTHON: "C:\\Python35" PYTHON_VERSION: "3.5.1" PYTHON_ARCH: "32" - # Use mocked-up GRPC for now - PYTHON: "C:\\Python35-x64" PYTHON_VERSION: "3.5.1" PYTHON_ARCH: "64" - # Use mocked-up GRPC for now install: - ECHO "Filesystem root:" @@ -85,7 +81,6 @@ build_script: - "%CMD_IN_ENV% python setup.py build" test_script: - - "set PYTHONPATH=%GRPC_PATH%" - "%CMD_IN_ENV% pip list" # Run the project tests - "%CMD_IN_ENV% python setup.py nosetests" diff --git a/docs/bigtable-client-intro.rst b/docs/bigtable-client-intro.rst index db04ffa0e0c1..5f30656db08b 100644 --- a/docs/bigtable-client-intro.rst +++ b/docs/bigtable-client-intro.rst @@ -20,11 +20,9 @@ Long-lived Defaults ------------------- When creating a :class:`Client `, the -``user_agent`` and ``timeout_seconds`` arguments have sensible -defaults -(:data:`DEFAULT_USER_AGENT ` and -:data:`DEFAULT_TIMEOUT_SECONDS `). -However, you may over-ride them and these will be used throughout all API +``user_agent`` argument has sensible a default +(:data:`DEFAULT_USER_AGENT `). +However, you may over-ride it and the value will be used throughout all API requests made with the ``client`` you create. Configuration diff --git a/gcloud/_helpers.py b/gcloud/_helpers.py index 4b06eca8da89..c74e8637f7e7 100644 --- a/gcloud/_helpers.py +++ b/gcloud/_helpers.py @@ -30,9 +30,13 @@ except ImportError: app_identity = None try: - from grpc.beta import implementations + from google.gax.grpc import exc_to_code as beta_exc_to_code + import grpc + from grpc._channel import _Rendezvous except ImportError: # pragma: NO COVER - implementations = None + beta_exc_to_code = None + grpc = None + _Rendezvous = Exception import six from six.moves.http_client import HTTPConnection from six.moves import configparser @@ -572,10 +576,10 @@ def __call__(self, unused_context, callback): callback(headers, None) -def make_stub(credentials, user_agent, stub_factory, host, port): +def make_stub(credentials, user_agent, stub_class, host, port): """Makes a stub for an RPC service. - Uses / depends on the beta implementation of gRPC. + Uses / depends on gRPC. :type credentials: :class:`oauth2client.client.OAuth2Credentials` :param credentials: The OAuth2 Credentials to use for creating @@ -584,9 +588,8 @@ def make_stub(credentials, user_agent, stub_factory, host, port): :type user_agent: str :param user_agent: (Optional) The user agent to be used with API requests. - :type stub_factory: callable - :param stub_factory: A factory which will create a gRPC stub for - a given service. + :type stub_class: type + :param stub_class: A gRPC stub type for a given service. :type host: str :param host: The host for the service. @@ -594,19 +597,35 @@ def make_stub(credentials, user_agent, stub_factory, host, port): :type port: int :param port: The port for the service. - :rtype: :class:`grpc.beta._stub._AutoIntermediary` + :rtype: object, instance of ``stub_class`` :returns: The stub object used to make gRPC requests to a given API. """ # Leaving the first argument to ssl_channel_credentials() as None # loads root certificates from `grpc/_adapter/credentials/roots.pem`. - transport_creds = implementations.ssl_channel_credentials(None, None, None) + transport_creds = grpc.ssl_channel_credentials() custom_metadata_plugin = MetadataPlugin(credentials, user_agent) - auth_creds = implementations.metadata_call_credentials( + auth_creds = grpc.metadata_call_credentials( custom_metadata_plugin, name='google_creds') - channel_creds = implementations.composite_channel_credentials( + channel_creds = grpc.composite_channel_credentials( transport_creds, auth_creds) - channel = implementations.secure_channel(host, port, channel_creds) - return stub_factory(channel) + target = '%s:%d' % (host, port) + channel = grpc.secure_channel(target, channel_creds) + return stub_class(channel) + + +def exc_to_code(exc): + """Retrieves the status code from a gRPC exception. + + :type exc: :class:`Exception` + :param exc: An exception from gRPC beta or stable. + + :rtype: :class:`grpc.StatusCode` + :returns: The status code attached to the exception. + """ + if isinstance(exc, _Rendezvous): + return exc.code() + else: + return beta_exc_to_code(exc) try: diff --git a/gcloud/_testing.py b/gcloud/_testing.py index 2212876431d1..d3a1a268b5b4 100644 --- a/gcloud/_testing.py +++ b/gcloud/_testing.py @@ -58,23 +58,19 @@ def __init__(self, **kw): self.__dict__.update(kw) def _make_grpc_error(self, status_code): - from grpc.framework.interfaces.face.face import AbortionError + from grpc._channel import _Rendezvous + from grpc._channel import _RPCState - class _DummyException(AbortionError): - code = status_code - - def __init__(self): - super(_DummyException, self).__init__( - None, None, self.code, None) - - return _DummyException() + details = 'Some error details.' + exc_state = _RPCState((), None, None, status_code, details) + return _Rendezvous(exc_state, None, None, None) def _make_grpc_not_found(self): - from grpc.beta.interfaces import StatusCode + from grpc import StatusCode return self._make_grpc_error(StatusCode.NOT_FOUND) def _make_grpc_failed_precondition(self): - from grpc.beta.interfaces import StatusCode + from grpc import StatusCode return self._make_grpc_error(StatusCode.FAILED_PRECONDITION) diff --git a/gcloud/bigtable/__init__.py b/gcloud/bigtable/__init__.py index 6b4e26681733..ff3eb0aa709a 100644 --- a/gcloud/bigtable/__init__.py +++ b/gcloud/bigtable/__init__.py @@ -33,6 +33,6 @@ """ try: - import grpc.beta.implementations + import grpc except ImportError as exc: # pragma: NO COVER raise ImportError(_ERR_MSG, exc) diff --git a/gcloud/bigtable/_testing.py b/gcloud/bigtable/_testing.py index d5f13c15d3a6..402a29ba92eb 100644 --- a/gcloud/bigtable/_testing.py +++ b/gcloud/bigtable/_testing.py @@ -21,16 +21,6 @@ class _FakeStub(object): def __init__(self, *results): self.results = results self.method_calls = [] - self._entered = 0 - self._exited = [] - - def __enter__(self): - self._entered += 1 - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self._exited.append((exc_type, exc_val, exc_tb)) - return True def __getattr__(self, name): # We need not worry about attributes set in constructor @@ -41,17 +31,16 @@ def __getattr__(self, name): class _MethodMock(object): """Mock for API method attached to a gRPC stub. - In the beta implementation, these are of type. - :class:`grpc.framework.crust.implementations._UnaryUnaryMultiCallable` + These are of type :class:`grpc._channel._UnaryUnaryMultiCallable`. """ - def __init__(self, name, factory): + def __init__(self, name, stub): self._name = name - self._factory = factory + self._stub = stub def __call__(self, *args, **kwargs): """Sync method meant to mock a gRPC stub request.""" - self._factory.method_calls.append((self._name, args, kwargs)) - curr_result, self._factory.results = (self._factory.results[0], - self._factory.results[1:]) + self._stub.method_calls.append((self._name, args, kwargs)) + curr_result, self._stub.results = (self._stub.results[0], + self._stub.results[1:]) return curr_result diff --git a/gcloud/bigtable/client.py b/gcloud/bigtable/client.py index e0829c0d8a9b..de124f489c63 100644 --- a/gcloud/bigtable/client.py +++ b/gcloud/bigtable/client.py @@ -30,19 +30,10 @@ from pkg_resources import get_distribution from gcloud._helpers import make_stub - -from gcloud.bigtable._generated import ( - bigtable_instance_admin_pb2 as instance_admin_v2_pb2) -# V1 table admin service -from gcloud.bigtable._generated import ( - bigtable_table_admin_pb2 as table_admin_v2_pb2) -# V1 data service -from gcloud.bigtable._generated import ( - bigtable_pb2 as data_v2_pb2) - -from gcloud.bigtable._generated import ( - operations_grpc_pb2 as operations_grpc_v2_pb2) - +from gcloud.bigtable._generated import bigtable_instance_admin_pb2 +from gcloud.bigtable._generated import bigtable_pb2 +from gcloud.bigtable._generated import bigtable_table_admin_pb2 +from gcloud.bigtable._generated import operations_grpc_pb2 from gcloud.bigtable.cluster import DEFAULT_SERVE_NODES from gcloud.bigtable.instance import Instance from gcloud.bigtable.instance import _EXISTING_INSTANCE_LOCATION_ID @@ -51,29 +42,23 @@ from gcloud.credentials import get_credentials -TABLE_STUB_FACTORY_V2 = ( - table_admin_v2_pb2.beta_create_BigtableTableAdmin_stub) -TABLE_ADMIN_HOST_V2 = 'bigtableadmin.googleapis.com' +TABLE_ADMIN_HOST = 'bigtableadmin.googleapis.com' """Table Admin API request host.""" -TABLE_ADMIN_PORT_V2 = 443 +TABLE_ADMIN_PORT = 443 """Table Admin API request port.""" -INSTANCE_STUB_FACTORY_V2 = ( - instance_admin_v2_pb2.beta_create_BigtableInstanceAdmin_stub) -INSTANCE_ADMIN_HOST_V2 = 'bigtableadmin.googleapis.com' +INSTANCE_ADMIN_HOST = 'bigtableadmin.googleapis.com' """Cluster Admin API request host.""" -INSTANCE_ADMIN_PORT_V2 = 443 +INSTANCE_ADMIN_PORT = 443 """Cluster Admin API request port.""" -DATA_STUB_FACTORY_V2 = data_v2_pb2.beta_create_Bigtable_stub -DATA_API_HOST_V2 = 'bigtable.googleapis.com' +DATA_API_HOST = 'bigtable.googleapis.com' """Data API request host.""" -DATA_API_PORT_V2 = 443 +DATA_API_PORT = 443 """Data API request port.""" -OPERATIONS_STUB_FACTORY_V2 = operations_grpc_v2_pb2.beta_create_Operations_stub -OPERATIONS_API_HOST_V2 = INSTANCE_ADMIN_HOST_V2 -OPERATIONS_API_PORT_V2 = INSTANCE_ADMIN_PORT_V2 +OPERATIONS_API_HOST = INSTANCE_ADMIN_HOST +OPERATIONS_API_PORT = INSTANCE_ADMIN_PORT ADMIN_SCOPE = 'https://www.googleapis.com/auth/bigtable.admin' """Scope for interacting with the Cluster Admin and Table Admin APIs.""" @@ -82,14 +67,70 @@ READ_ONLY_SCOPE = 'https://www.googleapis.com/auth/bigtable.data.readonly' """Scope for reading table data.""" -DEFAULT_TIMEOUT_SECONDS = 10 -"""The default timeout to use for API requests.""" - DEFAULT_USER_AGENT = 'gcloud-python/{0}'.format( get_distribution('gcloud').version) """The default user agent for API requests.""" +def _make_data_stub(client): + """Creates gRPC stub to make requests to the Data API. + + :type client: :class:`Client` + :param client: The client that will hold the stub. + + :rtype: :class:`._generated.bigtable_pb2.BigtableStub` + :returns: A gRPC stub object. + """ + return make_stub(client.credentials, client.user_agent, + bigtable_pb2.BigtableStub, + DATA_API_HOST, DATA_API_PORT) + + +def _make_instance_stub(client): + """Creates gRPC stub to make requests to the Instance Admin API. + + :type client: :class:`Client` + :param client: The client that will hold the stub. + + :rtype: :class:`.bigtable_instance_admin_pb2.BigtableInstanceAdminStub` + :returns: A gRPC stub object. + """ + return make_stub(client.credentials, client.user_agent, + bigtable_instance_admin_pb2.BigtableInstanceAdminStub, + INSTANCE_ADMIN_HOST, INSTANCE_ADMIN_PORT) + + +def _make_operations_stub(client): + """Creates gRPC stub to make requests to the Operations API. + + These are for long-running operations of the Instance Admin API, + hence the host and port matching. + + :type client: :class:`Client` + :param client: The client that will hold the stub. + + :rtype: :class:`._generated.operations_grpc_pb2.OperationsStub` + :returns: A gRPC stub object. + """ + return make_stub(client.credentials, client.user_agent, + operations_grpc_pb2.OperationsStub, + OPERATIONS_API_HOST, OPERATIONS_API_PORT) + + +def _make_table_stub(client): + """Creates gRPC stub to make requests to the Table Admin API. + + :type client: :class:`Client` + :param client: The client that will hold the stub. + + :rtype: :class:`.bigtable_instance_admin_pb2.BigtableTableAdminStub` + :returns: A gRPC stub object. + """ + return make_stub(client.credentials, client.user_agent, + bigtable_table_admin_pb2.BigtableTableAdminStub, + TABLE_ADMIN_HOST, TABLE_ADMIN_PORT) + + class Client(_ClientFactoryMixin, _ClientProjectMixin): """Client for interacting with Google Cloud Bigtable API. @@ -124,18 +165,16 @@ class Client(_ClientFactoryMixin, _ClientProjectMixin): :param user_agent: (Optional) The user agent to be used with API request. Defaults to :const:`DEFAULT_USER_AGENT`. - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. If not - passed, defaults to - :const:`DEFAULT_TIMEOUT_SECONDS`. - :raises: :class:`ValueError ` if both ``read_only`` and ``admin`` are :data:`True` """ + _instance_stub_internal = None + _operations_stub_internal = None + _table_stub_internal = None + def __init__(self, project=None, credentials=None, - read_only=False, admin=False, user_agent=DEFAULT_USER_AGENT, - timeout_seconds=DEFAULT_TIMEOUT_SECONDS): + read_only=False, admin=False, user_agent=DEFAULT_USER_AGENT): _ClientProjectMixin.__init__(self, project=project) if credentials is None: credentials = get_credentials() @@ -160,13 +199,13 @@ def __init__(self, project=None, credentials=None, pass self._credentials = credentials self.user_agent = user_agent - self.timeout_seconds = timeout_seconds - # These will be set in start(). - self._data_stub_internal = None - self._instance_stub_internal = None - self._operations_stub_internal = None - self._table_stub_internal = None + # Create gRPC stubs for making requests. + self._data_stub = _make_data_stub(self) + if self._admin: + self._instance_stub_internal = _make_instance_stub(self) + self._operations_stub_internal = _make_operations_stub(self) + self._table_stub_internal = _make_table_stub(self) def copy(self): """Make a copy of this client. @@ -185,7 +224,6 @@ def copy(self): READ_ONLY_SCOPE in copied_creds.scopes, self._admin, self.user_agent, - self.timeout_seconds, ) @property @@ -217,24 +255,11 @@ def project_name(self): """ return 'projects/' + self.project - @property - def _data_stub(self): - """Getter for the gRPC stub used for the Data API. - - :rtype: :class:`grpc.beta._stub._AutoIntermediary` - :returns: A gRPC stub object. - :raises: :class:`ValueError ` if the current - client has not been :meth:`start`-ed. - """ - if self._data_stub_internal is None: - raise ValueError('Client has not been started.') - return self._data_stub_internal - @property def _instance_stub(self): """Getter for the gRPC stub used for the Instance Admin API. - :rtype: :class:`grpc.beta._stub._AutoIntermediary` + :rtype: :class:`.bigtable_instance_admin_pb2.BigtableInstanceAdminStub` :returns: A gRPC stub object. :raises: :class:`ValueError ` if the current client is not an admin client or if it has not been @@ -242,15 +267,13 @@ def _instance_stub(self): """ if not self._admin: raise ValueError('Client is not an admin client.') - if self._instance_stub_internal is None: - raise ValueError('Client has not been started.') return self._instance_stub_internal @property def _operations_stub(self): """Getter for the gRPC stub used for the Operations API. - :rtype: :class:`grpc.beta._stub._AutoIntermediary` + :rtype: :class:`._generated.operations_grpc_pb2.OperationsStub` :returns: A gRPC stub object. :raises: :class:`ValueError ` if the current client is not an admin client or if it has not been @@ -258,15 +281,13 @@ def _operations_stub(self): """ if not self._admin: raise ValueError('Client is not an admin client.') - if self._operations_stub_internal is None: - raise ValueError('Client has not been started.') return self._operations_stub_internal @property def _table_stub(self): """Getter for the gRPC stub used for the Table Admin API. - :rtype: :class:`grpc.beta._stub._AutoIntermediary` + :rtype: :class:`.bigtable_instance_admin_pb2.BigtableTableAdminStub` :returns: A gRPC stub object. :raises: :class:`ValueError ` if the current client is not an admin client or if it has not been @@ -274,113 +295,8 @@ def _table_stub(self): """ if not self._admin: raise ValueError('Client is not an admin client.') - if self._table_stub_internal is None: - raise ValueError('Client has not been started.') return self._table_stub_internal - def _make_data_stub(self): - """Creates gRPC stub to make requests to the Data API. - - :rtype: :class:`grpc.beta._stub._AutoIntermediary` - :returns: A gRPC stub object. - """ - return make_stub(self.credentials, self.user_agent, - DATA_STUB_FACTORY_V2, DATA_API_HOST_V2, - DATA_API_PORT_V2) - - def _make_instance_stub(self): - """Creates gRPC stub to make requests to the Instance Admin API. - - :rtype: :class:`grpc.beta._stub._AutoIntermediary` - :returns: A gRPC stub object. - """ - return make_stub(self.credentials, self.user_agent, - INSTANCE_STUB_FACTORY_V2, INSTANCE_ADMIN_HOST_V2, - INSTANCE_ADMIN_PORT_V2) - - def _make_operations_stub(self): - """Creates gRPC stub to make requests to the Operations API. - - These are for long-running operations of the Instance Admin API, - hence the host and port matching. - - :rtype: :class:`grpc.beta._stub._AutoIntermediary` - :returns: A gRPC stub object. - """ - return make_stub(self.credentials, self.user_agent, - OPERATIONS_STUB_FACTORY_V2, OPERATIONS_API_HOST_V2, - OPERATIONS_API_PORT_V2) - - def _make_table_stub(self): - """Creates gRPC stub to make requests to the Table Admin API. - - :rtype: :class:`grpc.beta._stub._AutoIntermediary` - :returns: A gRPC stub object. - """ - return make_stub(self.credentials, self.user_agent, - TABLE_STUB_FACTORY_V2, TABLE_ADMIN_HOST_V2, - TABLE_ADMIN_PORT_V2) - - def is_started(self): - """Check if the client has been started. - - :rtype: bool - :returns: Boolean indicating if the client has been started. - """ - return self._data_stub_internal is not None - - def start(self): - """Prepare the client to make requests. - - Activates gRPC contexts for making requests to the Bigtable - Service(s). - """ - if self.is_started(): - return - - # NOTE: We __enter__ the stubs more-or-less permanently. This is - # because only after entering the context managers is the - # connection created. We don't want to immediately close - # those connections since the client will make many - # requests with it over HTTP/2. - self._data_stub_internal = self._make_data_stub() - self._data_stub_internal.__enter__() - if self._admin: - self._instance_stub_internal = self._make_instance_stub() - self._operations_stub_internal = self._make_operations_stub() - self._table_stub_internal = self._make_table_stub() - - self._instance_stub_internal.__enter__() - self._operations_stub_internal.__enter__() - self._table_stub_internal.__enter__() - - def __enter__(self): - """Starts the client as a context manager.""" - self.start() - return self - - def stop(self): - """Closes all the open gRPC clients.""" - if not self.is_started(): - return - - # When exit-ing, we pass None as the exception type, value and - # traceback to __exit__. - self._data_stub_internal.__exit__(None, None, None) - if self._admin: - self._instance_stub_internal.__exit__(None, None, None) - self._operations_stub_internal.__exit__(None, None, None) - self._table_stub_internal.__exit__(None, None, None) - - self._data_stub_internal = None - self._instance_stub_internal = None - self._operations_stub_internal = None - self._table_stub_internal = None - - def __exit__(self, exc_type, exc_val, exc_t): - """Stops the client as a context manager.""" - self.stop() - def instance(self, instance_id, location=_EXISTING_INSTANCE_LOCATION_ID, display_name=None, serve_nodes=DEFAULT_SERVE_NODES): """Factory to create a instance associated with this client. @@ -417,11 +333,10 @@ def list_instances(self): :class:`.Instance` objects returned and the second is a list of strings (the failed locations in the request). """ - request_pb = instance_admin_v2_pb2.ListInstancesRequest( + request_pb = bigtable_instance_admin_pb2.ListInstancesRequest( parent=self.project_name) - response = self._instance_stub.ListInstances( - request_pb, self.timeout_seconds) + response = self._instance_stub.ListInstances(request_pb) instances = [Instance.from_pb(instance_pb, self) for instance_pb in response.instances] diff --git a/gcloud/bigtable/cluster.py b/gcloud/bigtable/cluster.py index 993f622afdd0..db43be2e34c1 100644 --- a/gcloud/bigtable/cluster.py +++ b/gcloud/bigtable/cluster.py @@ -152,8 +152,7 @@ def finished(self): request_pb = operations_pb2.GetOperationRequest(name=operation_name) # We expect a `google.longrunning.operations_pb2.Operation`. client = self._cluster._instance._client - operation_pb = client._operations_stub.GetOperation( - request_pb, client.timeout_seconds) + operation_pb = client._operations_stub.GetOperation(request_pb) if operation_pb.done: self._complete = True @@ -290,7 +289,7 @@ def reload(self): request_pb = messages_v2_pb2.GetClusterRequest(name=self.name) # We expect a `._generated.instance_pb2.Cluster`. cluster_pb = self._instance._client._instance_stub.GetCluster( - request_pb, self._instance._client.timeout_seconds) + request_pb) # NOTE: _update_from_pb does not check that the project, instance and # cluster ID on the response match the request. @@ -319,7 +318,7 @@ def create(self): request_pb = _prepare_create_request(self) # We expect a `google.longrunning.operations_pb2.Operation`. operation_pb = self._instance._client._instance_stub.CreateCluster( - request_pb, self._instance._client.timeout_seconds) + request_pb) op_id = _process_operation(operation_pb) return Operation('create', op_id, cluster=self) @@ -348,7 +347,7 @@ def update(self): ) # Ignore expected `._generated.instance_pb2.Cluster`. operation_pb = self._instance._client._instance_stub.UpdateCluster( - request_pb, self._instance._client.timeout_seconds) + request_pb) op_id = _process_operation(operation_pb) return Operation('update', op_id, cluster=self) @@ -380,5 +379,4 @@ def delete(self): """ request_pb = messages_v2_pb2.DeleteClusterRequest(name=self.name) # We expect a `google.protobuf.empty_pb2.Empty` - self._instance._client._instance_stub.DeleteCluster( - request_pb, self._instance._client.timeout_seconds) + self._instance._client._instance_stub.DeleteCluster(request_pb) diff --git a/gcloud/bigtable/column_family.py b/gcloud/bigtable/column_family.py index 166f96ecdb72..682375c8dcfd 100644 --- a/gcloud/bigtable/column_family.py +++ b/gcloud/bigtable/column_family.py @@ -271,8 +271,7 @@ def create(self): # We expect a `.table_v2_pb2.ColumnFamily`. We ignore it since the only # data it contains are the GC rule and the column family ID already # stored on this instance. - client._table_stub.ModifyColumnFamilies(request_pb, - client.timeout_seconds) + client._table_stub.ModifyColumnFamilies(request_pb) def update(self): """Update this column family. @@ -292,8 +291,7 @@ def update(self): # We expect a `.table_v2_pb2.ColumnFamily`. We ignore it since the only # data it contains are the GC rule and the column family ID already # stored on this instance. - client._table_stub.ModifyColumnFamilies(request_pb, - client.timeout_seconds) + client._table_stub.ModifyColumnFamilies(request_pb) def delete(self): """Delete this column family.""" @@ -304,8 +302,7 @@ def delete(self): drop=True) client = self._table._instance._client # We expect a `google.protobuf.empty_pb2.Empty` - client._table_stub.ModifyColumnFamilies(request_pb, - client.timeout_seconds) + client._table_stub.ModifyColumnFamilies(request_pb) def _gc_rule_from_pb(gc_rule_pb): diff --git a/gcloud/bigtable/instance.py b/gcloud/bigtable/instance.py index e83c4eb8c698..a70b276812fd 100644 --- a/gcloud/bigtable/instance.py +++ b/gcloud/bigtable/instance.py @@ -181,7 +181,7 @@ def finished(self): request_pb = operations_pb2.GetOperationRequest(name=operation_name) # We expect a `google.longrunning.operations_pb2.Operation`. operation_pb = self._instance._client._operations_stub.GetOperation( - request_pb, self._instance._client.timeout_seconds) + request_pb) if operation_pb.done: self._complete = True @@ -329,8 +329,7 @@ def reload(self): """Reload the metadata for this instance.""" request_pb = messages_v2_pb2.GetInstanceRequest(name=self.name) # We expect `data_v2_pb2.Instance`. - instance_pb = self._client._instance_stub.GetInstance( - request_pb, self._client.timeout_seconds) + instance_pb = self._client._instance_stub.GetInstance(request_pb) # NOTE: _update_from_pb does not check that the project and # instance ID on the response match the request. @@ -358,8 +357,7 @@ def create(self): """ request_pb = _prepare_create_request(self) # We expect a `google.longrunning.operations_pb2.Operation`. - operation_pb = self._client._instance_stub.CreateInstance( - request_pb, self._client.timeout_seconds) + operation_pb = self._client._instance_stub.CreateInstance(request_pb) op_id, loc_id, op_begin = _process_operation(operation_pb) return Operation('create', op_id, op_begin, loc_id, instance=self) @@ -383,8 +381,7 @@ def update(self): display_name=self.display_name, ) # Ignore the expected `data_v2_pb2.Instance`. - self._client._instance_stub.UpdateInstance( - request_pb, self._client.timeout_seconds) + self._client._instance_stub.UpdateInstance(request_pb) def delete(self): """Delete this instance. @@ -415,8 +412,7 @@ def delete(self): """ request_pb = messages_v2_pb2.DeleteInstanceRequest(name=self.name) # We expect a `google.protobuf.empty_pb2.Empty` - self._client._instance_stub.DeleteInstance( - request_pb, self._client.timeout_seconds) + self._client._instance_stub.DeleteInstance(request_pb) def cluster(self, cluster_id, serve_nodes=3): """Factory to create a cluster associated with this client. @@ -444,7 +440,7 @@ def list_clusters(self): request_pb = messages_v2_pb2.ListClustersRequest(parent=self.name) # We expect a `.cluster_messages_v1_pb2.ListClustersResponse` list_clusters_response = self._client._instance_stub.ListClusters( - request_pb, self._client.timeout_seconds) + request_pb) failed_locations = [ location for location in list_clusters_response.failed_locations] @@ -473,8 +469,7 @@ def list_tables(self): """ request_pb = table_messages_v2_pb2.ListTablesRequest(parent=self.name) # We expect a `table_messages_v2_pb2.ListTablesResponse` - table_list_pb = self._client._table_stub.ListTables( - request_pb, self._client.timeout_seconds) + table_list_pb = self._client._table_stub.ListTables(request_pb) result = [] for table_pb in table_list_pb.tables: diff --git a/gcloud/bigtable/row.py b/gcloud/bigtable/row.py index f04df29d381b..e7a1b7bc491d 100644 --- a/gcloud/bigtable/row.py +++ b/gcloud/bigtable/row.py @@ -396,7 +396,7 @@ def commit(self): ) # We expect a `google.protobuf.empty_pb2.Empty` client = self._table._instance._client - client._data_stub.MutateRow(request_pb, client.timeout_seconds) + client._data_stub.MutateRow(request_pb) self.clear() def clear(self): @@ -513,8 +513,7 @@ def commit(self): ) # We expect a `.messages_v2_pb2.CheckAndMutateRowResponse` client = self._table._instance._client - resp = client._data_stub.CheckAndMutateRow( - request_pb, client.timeout_seconds) + resp = client._data_stub.CheckAndMutateRow(request_pb) self.clear() return resp.predicate_matched @@ -801,8 +800,7 @@ def commit(self): ) # We expect a `.data_v2_pb2.Row` client = self._table._instance._client - row_response = client._data_stub.ReadModifyWriteRow( - request_pb, client.timeout_seconds) + row_response = client._data_stub.ReadModifyWriteRow(request_pb) # Reset modifications after commit-ing request. self.clear() diff --git a/gcloud/bigtable/row_data.py b/gcloud/bigtable/row_data.py index 55f42e5be57f..e6b77508b4a7 100644 --- a/gcloud/bigtable/row_data.py +++ b/gcloud/bigtable/row_data.py @@ -183,8 +183,7 @@ class InvalidChunk(RuntimeError): class PartialRowsData(object): """Convenience wrapper for consuming a ``ReadRows`` streaming response. - :type response_iterator: - :class:`grpc.framework.alpha._reexport._CancellableIterator` + :type response_iterator: :class:`grpc._channel._Rendezvous` :param response_iterator: A streaming iterator returned from a ``ReadRows`` request. """ diff --git a/gcloud/bigtable/table.py b/gcloud/bigtable/table.py index 3052643ba032..b0e1d73d07fe 100644 --- a/gcloud/bigtable/table.py +++ b/gcloud/bigtable/table.py @@ -175,7 +175,7 @@ def create(self, initial_split_keys=None, column_families=()): ) client = self._instance._client # We expect a `._generated.table_pb2.Table` - client._table_stub.CreateTable(request_pb, client.timeout_seconds) + client._table_stub.CreateTable(request_pb) def delete(self): """Delete this table.""" @@ -183,7 +183,7 @@ def delete(self): name=self.name) client = self._instance._client # We expect a `google.protobuf.empty_pb2.Empty` - client._table_stub.DeleteTable(request_pb, client.timeout_seconds) + client._table_stub.DeleteTable(request_pb) def list_column_families(self): """List the column families owned by this table. @@ -200,8 +200,7 @@ def list_column_families(self): name=self.name) client = self._instance._client # We expect a `._generated.table_pb2.Table` - table_pb = client._table_stub.GetTable(request_pb, - client.timeout_seconds) + table_pb = client._table_stub.GetTable(request_pb) result = {} for column_family_id, value_pb in table_pb.column_families.items(): @@ -230,8 +229,7 @@ def read_row(self, row_key, filter_=None): request_pb = _create_row_request(self.name, row_key=row_key, filter_=filter_) client = self._instance._client - response_iterator = client._data_stub.ReadRows(request_pb, - client.timeout_seconds) + response_iterator = client._data_stub.ReadRows(request_pb) rows_data = PartialRowsData(response_iterator) rows_data.consume_all() if rows_data.state not in (rows_data.NEW_ROW, rows_data.START): @@ -274,8 +272,7 @@ def read_rows(self, start_key=None, end_key=None, limit=None, self.name, start_key=start_key, end_key=end_key, filter_=filter_, limit=limit) client = self._instance._client - response_iterator = client._data_stub.ReadRows(request_pb, - client.timeout_seconds) + response_iterator = client._data_stub.ReadRows(request_pb) # We expect an iterator of `data_messages_v2_pb2.ReadRowsResponse` return PartialRowsData(response_iterator) @@ -305,7 +302,7 @@ def sample_row_keys(self): samples would require space roughly equal to the difference in their ``offset_bytes`` fields. - :rtype: :class:`grpc.framework.alpha._reexport._CancellableIterator` + :rtype: :class:`grpc._channel._Rendezvous` :returns: A cancel-able iterator. Can be consumed by calling ``next()`` or by casting to a :class:`list` and can be cancelled by calling ``cancel()``. @@ -313,8 +310,7 @@ def sample_row_keys(self): request_pb = data_messages_v2_pb2.SampleRowKeysRequest( table_name=self.name) client = self._instance._client - response_iterator = client._data_stub.SampleRowKeys( - request_pb, client.timeout_seconds) + response_iterator = client._data_stub.SampleRowKeys(request_pb) return response_iterator diff --git a/gcloud/bigtable/test_client.py b/gcloud/bigtable/test_client.py index 1eb02a9f67b7..f071f8f1bf61 100644 --- a/gcloud/bigtable/test_client.py +++ b/gcloud/bigtable/test_client.py @@ -16,12 +16,155 @@ import unittest +class Test__make_data_stub(unittest.TestCase): + + def _callFUT(self, client): + from gcloud.bigtable.client import _make_data_stub + return _make_data_stub(client) + + def test_it(self): + from gcloud._testing import _Monkey + from gcloud.bigtable import client as MUT + + credentials = _Credentials() + user_agent = 'you-sir-age-int' + client = _Client(credentials, user_agent) + + fake_stub = object() + make_stub_args = [] + + def mock_make_stub(*args): + make_stub_args.append(args) + return fake_stub + + with _Monkey(MUT, make_stub=mock_make_stub): + result = self._callFUT(client) + + self.assertIs(result, fake_stub) + self.assertEqual(make_stub_args, [ + ( + client.credentials, + client.user_agent, + MUT.bigtable_pb2.BigtableStub, + MUT.DATA_API_HOST, + MUT.DATA_API_PORT, + ), + ]) + + +class Test__make_instance_stub(unittest.TestCase): + + def _callFUT(self, client): + from gcloud.bigtable.client import _make_instance_stub + return _make_instance_stub(client) + + def test_it(self): + from gcloud._testing import _Monkey + from gcloud.bigtable import client as MUT + + credentials = _Credentials() + user_agent = 'you-sir-age-int' + client = _Client(credentials, user_agent) + + fake_stub = object() + make_stub_args = [] + + def mock_make_stub(*args): + make_stub_args.append(args) + return fake_stub + + with _Monkey(MUT, make_stub=mock_make_stub): + result = self._callFUT(client) + + self.assertIs(result, fake_stub) + self.assertEqual(make_stub_args, [ + ( + client.credentials, + client.user_agent, + MUT.bigtable_instance_admin_pb2.BigtableInstanceAdminStub, + MUT.INSTANCE_ADMIN_HOST, + MUT.INSTANCE_ADMIN_PORT, + ), + ]) + + +class Test__make_operations_stub(unittest.TestCase): + + def _callFUT(self, client): + from gcloud.bigtable.client import _make_operations_stub + return _make_operations_stub(client) + + def test_it(self): + from gcloud._testing import _Monkey + from gcloud.bigtable import client as MUT + + credentials = _Credentials() + user_agent = 'you-sir-age-int' + client = _Client(credentials, user_agent) + + fake_stub = object() + make_stub_args = [] + + def mock_make_stub(*args): + make_stub_args.append(args) + return fake_stub + + with _Monkey(MUT, make_stub=mock_make_stub): + result = self._callFUT(client) + + self.assertIs(result, fake_stub) + self.assertEqual(make_stub_args, [ + ( + client.credentials, + client.user_agent, + MUT.operations_grpc_pb2.OperationsStub, + MUT.OPERATIONS_API_HOST, + MUT.OPERATIONS_API_PORT, + ), + ]) + + +class Test__make_table_stub(unittest.TestCase): + + def _callFUT(self, client): + from gcloud.bigtable.client import _make_table_stub + return _make_table_stub(client) + + def test_it(self): + from gcloud._testing import _Monkey + from gcloud.bigtable import client as MUT + + credentials = _Credentials() + user_agent = 'you-sir-age-int' + client = _Client(credentials, user_agent) + + fake_stub = object() + make_stub_args = [] + + def mock_make_stub(*args): + make_stub_args.append(args) + return fake_stub + + with _Monkey(MUT, make_stub=mock_make_stub): + result = self._callFUT(client) + + self.assertIs(result, fake_stub) + self.assertEqual(make_stub_args, [ + ( + client.credentials, + client.user_agent, + MUT.bigtable_table_admin_pb2.BigtableTableAdminStub, + MUT.TABLE_ADMIN_HOST, + MUT.TABLE_ADMIN_PORT, + ), + ]) + + class TestClient(unittest.TestCase): PROJECT = 'PROJECT' INSTANCE_ID = 'instance-id' DISPLAY_NAME = 'display-name' - TIMEOUT_SECONDS = 80 USER_AGENT = 'you-sir-age-int' def _getTargetClass(self): @@ -31,18 +174,50 @@ def _getTargetClass(self): def _makeOne(self, *args, **kwargs): return self._getTargetClass()(*args, **kwargs) + def _makeOneWithMocks(self, *args, **kwargs): + from gcloud._testing import _Monkey + from gcloud.bigtable import client as MUT + + mock_make_data_stub = _MakeStubMock() + mock_make_instance_stub = _MakeStubMock() + mock_make_operations_stub = _MakeStubMock() + mock_make_table_stub = _MakeStubMock() + with _Monkey(MUT, _make_data_stub=mock_make_data_stub, + _make_instance_stub=mock_make_instance_stub, + _make_operations_stub=mock_make_operations_stub, + _make_table_stub=mock_make_table_stub): + return self._makeOne(*args, **kwargs) + def _constructor_test_helper(self, expected_scopes, creds, read_only=False, admin=False, - user_agent=None, timeout_seconds=None, - expected_creds=None): + user_agent=None, expected_creds=None): + from gcloud._testing import _Monkey from gcloud.bigtable import client as MUT user_agent = user_agent or MUT.DEFAULT_USER_AGENT - timeout_seconds = timeout_seconds or MUT.DEFAULT_TIMEOUT_SECONDS - client = self._makeOne(project=self.PROJECT, credentials=creds, - read_only=read_only, admin=admin, - user_agent=user_agent, - timeout_seconds=timeout_seconds) + + mock_make_data_stub = _MakeStubMock() + mock_make_instance_stub = _MakeStubMock() + mock_make_operations_stub = _MakeStubMock() + mock_make_table_stub = _MakeStubMock() + with _Monkey(MUT, _make_data_stub=mock_make_data_stub, + _make_instance_stub=mock_make_instance_stub, + _make_operations_stub=mock_make_operations_stub, + _make_table_stub=mock_make_table_stub): + client = self._makeOne(project=self.PROJECT, credentials=creds, + read_only=read_only, admin=admin, + user_agent=user_agent) + + # Verify the mocks. + self.assertEqual(mock_make_data_stub.calls, [client]) + if admin: + self.assertSequenceEqual(mock_make_instance_stub.calls, [client]) + self.assertSequenceEqual(mock_make_operations_stub.calls, [client]) + self.assertSequenceEqual(mock_make_table_stub.calls, [client]) + else: + self.assertSequenceEqual(mock_make_instance_stub.calls, []) + self.assertSequenceEqual(mock_make_operations_stub.calls, []) + self.assertSequenceEqual(mock_make_table_stub.calls, []) expected_creds = expected_creds or creds self.assertTrue(client._credentials is expected_creds) @@ -50,13 +225,20 @@ def _constructor_test_helper(self, expected_scopes, creds, self.assertEqual(client._credentials.scopes, expected_scopes) self.assertEqual(client.project, self.PROJECT) - self.assertEqual(client.timeout_seconds, timeout_seconds) self.assertEqual(client.user_agent, user_agent) - # Check stubs are set (but null) - self.assertEqual(client._data_stub_internal, None) - self.assertEqual(client._instance_stub_internal, None) - self.assertEqual(client._operations_stub_internal, None) - self.assertEqual(client._table_stub_internal, None) + # Check gRPC stubs (or mocks of them) are set + self.assertIs(client._data_stub, mock_make_data_stub.result) + if admin: + self.assertIs(client._instance_stub_internal, + mock_make_instance_stub.result) + self.assertIs(client._operations_stub_internal, + mock_make_operations_stub.result) + self.assertIs(client._table_stub_internal, + mock_make_table_stub.result) + else: + self.assertIsNone(client._instance_stub_internal) + self.assertIsNone(client._operations_stub_internal) + self.assertIsNone(client._table_stub_internal) def test_constructor_default_scopes(self): from gcloud.bigtable import client as MUT @@ -65,16 +247,14 @@ def test_constructor_default_scopes(self): creds = _Credentials() self._constructor_test_helper(expected_scopes, creds) - def test_constructor_custom_user_agent_and_timeout(self): + def test_constructor_custom_user_agent(self): from gcloud.bigtable import client as MUT - CUSTOM_TIMEOUT_SECONDS = 1337 CUSTOM_USER_AGENT = 'custom-application' expected_scopes = [MUT.DATA_SCOPE] creds = _Credentials() self._constructor_test_helper(expected_scopes, creds, - user_agent=CUSTOM_USER_AGENT, - timeout_seconds=CUSTOM_TIMEOUT_SECONDS) + user_agent=CUSTOM_USER_AGENT) def test_constructor_with_admin(self): from gcloud.bigtable import client as MUT @@ -115,71 +295,46 @@ def test_constructor_credentials_wo_create_scoped(self): expected_scopes = None self._constructor_test_helper(expected_scopes, creds) - def _context_manager_helper(self): - credentials = _Credentials() - client = self._makeOne(project=self.PROJECT, credentials=credentials) - - def mock_start(): - client._data_stub_internal = object() - client.start = mock_start - - def mock_stop(): - client._data_stub_internal = None - client.stop = mock_stop - return client - - def test_context_manager(self): - client = self._context_manager_helper() - self.assertFalse(client.is_started()) - with client: - self.assertTrue(client.is_started()) - self.assertFalse(client.is_started()) - - def test_context_manager_as_keyword(self): - with self._context_manager_helper() as client: - self.assertIsNotNone(client) - - def test_context_manager_with_exception(self): - client = self._context_manager_helper() - self.assertFalse(client.is_started()) - - class DummyException(Exception): - pass - try: - with client: - self.assertTrue(client.is_started()) - raise DummyException() - except DummyException: - pass - self.assertFalse(client.is_started()) - def _copy_test_helper(self, read_only=False, admin=False): + from gcloud._testing import _Monkey + from gcloud.bigtable import client as MUT + credentials = _Credentials('value') - client = self._makeOne( + client = self._makeOneWithMocks( project=self.PROJECT, credentials=credentials, read_only=read_only, admin=admin, - timeout_seconds=self.TIMEOUT_SECONDS, user_agent=self.USER_AGENT) - # Put some fake stubs in place so that we can verify they - # don't get copied. - client._data_stub_internal = object() + # Put some fake stubs in place so that we can verify they don't + # get copied. In the admin=False case, only the data stub will + # not be None, so we over-ride all the internal values. + client._data_stub = object() client._instance_stub_internal = object() client._operations_stub_internal = object() client._table_stub_internal = object() - new_client = client.copy() + mock_make_data_stub = _MakeStubMock() + mock_make_instance_stub = _MakeStubMock() + mock_make_operations_stub = _MakeStubMock() + mock_make_table_stub = _MakeStubMock() + with _Monkey(MUT, _make_data_stub=mock_make_data_stub, + _make_instance_stub=mock_make_instance_stub, + _make_operations_stub=mock_make_operations_stub, + _make_table_stub=mock_make_table_stub): + new_client = client.copy() self.assertEqual(new_client._admin, client._admin) self.assertEqual(new_client._credentials, client._credentials) self.assertEqual(new_client.project, client.project) self.assertEqual(new_client.user_agent, client.user_agent) - self.assertEqual(new_client.timeout_seconds, client.timeout_seconds) # Make sure stubs are not preserved. - self.assertEqual(new_client._data_stub_internal, None) - self.assertEqual(new_client._instance_stub_internal, None) - self.assertEqual(new_client._operations_stub_internal, None) - self.assertEqual(new_client._table_stub_internal, None) + self.assertNotEqual(new_client._data_stub, client._data_stub) + self.assertNotEqual(new_client._instance_stub_internal, + client._instance_stub_internal) + self.assertNotEqual(new_client._operations_stub_internal, + client._operations_stub_internal) + self.assertNotEqual(new_client._table_stub_internal, + client._table_stub_internal) def test_copy(self): self._copy_test_helper() @@ -193,343 +348,64 @@ def test_copy_read_only(self): def test_credentials_getter(self): credentials = _Credentials() project = 'PROJECT' - client = self._makeOne(project=project, credentials=credentials) + client = self._makeOneWithMocks(project=project, + credentials=credentials) self.assertTrue(client.credentials is credentials) def test_project_name_property(self): credentials = _Credentials() project = 'PROJECT' - client = self._makeOne(project=project, credentials=credentials) + client = self._makeOneWithMocks(project=project, + credentials=credentials) project_name = 'projects/' + project self.assertEqual(client.project_name, project_name) - def test_data_stub_getter(self): - credentials = _Credentials() - project = 'PROJECT' - client = self._makeOne(project=project, credentials=credentials) - client._data_stub_internal = object() - self.assertTrue(client._data_stub is client._data_stub_internal) - - def test_data_stub_failure(self): - credentials = _Credentials() - project = 'PROJECT' - client = self._makeOne(project=project, credentials=credentials) - with self.assertRaises(ValueError): - getattr(client, '_data_stub') - def test_instance_stub_getter(self): credentials = _Credentials() project = 'PROJECT' - client = self._makeOne(project=project, credentials=credentials, - admin=True) - client._instance_stub_internal = object() - self.assertTrue( - client._instance_stub is client._instance_stub_internal) + client = self._makeOneWithMocks(project=project, + credentials=credentials, admin=True) + self.assertIs(client._instance_stub, client._instance_stub_internal) def test_instance_stub_non_admin_failure(self): credentials = _Credentials() project = 'PROJECT' - client = self._makeOne(project=project, credentials=credentials, - admin=False) - with self.assertRaises(ValueError): - getattr(client, '_instance_stub') - - def test_instance_stub_unset_failure(self): - credentials = _Credentials() - project = 'PROJECT' - client = self._makeOne(project=project, credentials=credentials, - admin=True) + client = self._makeOneWithMocks(project=project, + credentials=credentials, admin=False) with self.assertRaises(ValueError): getattr(client, '_instance_stub') def test_operations_stub_getter(self): credentials = _Credentials() project = 'PROJECT' - client = self._makeOne(project=project, credentials=credentials, - admin=True) - client._operations_stub_internal = object() - self.assertTrue(client._operations_stub is - client._operations_stub_internal) + client = self._makeOneWithMocks(project=project, + credentials=credentials, admin=True) + self.assertIs(client._operations_stub, + client._operations_stub_internal) def test_operations_stub_non_admin_failure(self): credentials = _Credentials() project = 'PROJECT' - client = self._makeOne(project=project, credentials=credentials, - admin=False) - with self.assertRaises(ValueError): - getattr(client, '_operations_stub') - - def test_operations_stub_unset_failure(self): - credentials = _Credentials() - project = 'PROJECT' - client = self._makeOne(project=project, credentials=credentials, - admin=True) + client = self._makeOneWithMocks(project=project, + credentials=credentials, admin=False) with self.assertRaises(ValueError): getattr(client, '_operations_stub') def test_table_stub_getter(self): credentials = _Credentials() project = 'PROJECT' - client = self._makeOne(project=project, credentials=credentials, - admin=True) - client._table_stub_internal = object() - self.assertTrue(client._table_stub is client._table_stub_internal) + client = self._makeOneWithMocks(project=project, + credentials=credentials, admin=True) + self.assertIs(client._table_stub, client._table_stub_internal) def test_table_stub_non_admin_failure(self): credentials = _Credentials() project = 'PROJECT' - client = self._makeOne(project=project, credentials=credentials, - admin=False) - with self.assertRaises(ValueError): - getattr(client, '_table_stub') - - def test_table_stub_unset_failure(self): - credentials = _Credentials() - project = 'PROJECT' - client = self._makeOne(project=project, credentials=credentials, - admin=True) + client = self._makeOneWithMocks(project=project, + credentials=credentials, admin=False) with self.assertRaises(ValueError): getattr(client, '_table_stub') - def test__make_data_stub(self): - from gcloud._testing import _Monkey - from gcloud.bigtable import client as MUT - from gcloud.bigtable.client import DATA_API_HOST_V2 - from gcloud.bigtable.client import DATA_API_PORT_V2 - from gcloud.bigtable.client import DATA_STUB_FACTORY_V2 - - credentials = _Credentials() - project = 'PROJECT' - client = self._makeOne(project=project, credentials=credentials) - - fake_stub = object() - make_stub_args = [] - - def mock_make_stub(*args): - make_stub_args.append(args) - return fake_stub - - with _Monkey(MUT, make_stub=mock_make_stub): - result = client._make_data_stub() - - self.assertTrue(result is fake_stub) - self.assertEqual(make_stub_args, [ - ( - client.credentials, - client.user_agent, - DATA_STUB_FACTORY_V2, - DATA_API_HOST_V2, - DATA_API_PORT_V2, - ), - ]) - - def test__make_instance_stub(self): - from gcloud._testing import _Monkey - from gcloud.bigtable import client as MUT - from gcloud.bigtable.client import INSTANCE_ADMIN_HOST_V2 - from gcloud.bigtable.client import INSTANCE_ADMIN_PORT_V2 - from gcloud.bigtable.client import INSTANCE_STUB_FACTORY_V2 - - credentials = _Credentials() - project = 'PROJECT' - client = self._makeOne(project=project, credentials=credentials) - - fake_stub = object() - make_stub_args = [] - - def mock_make_stub(*args): - make_stub_args.append(args) - return fake_stub - - with _Monkey(MUT, make_stub=mock_make_stub): - result = client._make_instance_stub() - - self.assertTrue(result is fake_stub) - self.assertEqual(make_stub_args, [ - ( - client.credentials, - client.user_agent, - INSTANCE_STUB_FACTORY_V2, - INSTANCE_ADMIN_HOST_V2, - INSTANCE_ADMIN_PORT_V2, - ), - ]) - - def test__make_operations_stub(self): - from gcloud._testing import _Monkey - from gcloud.bigtable import client as MUT - from gcloud.bigtable.client import OPERATIONS_API_HOST_V2 - from gcloud.bigtable.client import OPERATIONS_API_PORT_V2 - from gcloud.bigtable.client import OPERATIONS_STUB_FACTORY_V2 - - credentials = _Credentials() - project = 'PROJECT' - client = self._makeOne(project=project, credentials=credentials) - - fake_stub = object() - make_stub_args = [] - - def mock_make_stub(*args): - make_stub_args.append(args) - return fake_stub - - with _Monkey(MUT, make_stub=mock_make_stub): - result = client._make_operations_stub() - - self.assertTrue(result is fake_stub) - self.assertEqual(make_stub_args, [ - ( - client.credentials, - client.user_agent, - OPERATIONS_STUB_FACTORY_V2, - OPERATIONS_API_HOST_V2, - OPERATIONS_API_PORT_V2, - ), - ]) - - def test__make_table_stub(self): - from gcloud._testing import _Monkey - from gcloud.bigtable import client as MUT - from gcloud.bigtable.client import TABLE_ADMIN_HOST_V2 - from gcloud.bigtable.client import TABLE_ADMIN_PORT_V2 - from gcloud.bigtable.client import TABLE_STUB_FACTORY_V2 - - credentials = _Credentials() - project = 'PROJECT' - client = self._makeOne(project=project, credentials=credentials) - - fake_stub = object() - make_stub_args = [] - - def mock_make_stub(*args): - make_stub_args.append(args) - return fake_stub - - with _Monkey(MUT, make_stub=mock_make_stub): - result = client._make_table_stub() - - self.assertTrue(result is fake_stub) - self.assertEqual(make_stub_args, [ - ( - client.credentials, - client.user_agent, - TABLE_STUB_FACTORY_V2, - TABLE_ADMIN_HOST_V2, - TABLE_ADMIN_PORT_V2, - ), - ]) - - def test_is_started(self): - credentials = _Credentials() - project = 'PROJECT' - client = self._makeOne(project=project, credentials=credentials) - - self.assertFalse(client.is_started()) - client._data_stub_internal = object() - self.assertTrue(client.is_started()) - client._data_stub_internal = None - self.assertFalse(client.is_started()) - - def _start_method_helper(self, admin): - from gcloud._testing import _Monkey - from gcloud.bigtable._testing import _FakeStub - from gcloud.bigtable import client as MUT - - credentials = _Credentials() - project = 'PROJECT' - client = self._makeOne(project=project, credentials=credentials, - admin=admin) - - stub = _FakeStub() - make_stub_args = [] - - def mock_make_stub(*args): - make_stub_args.append(args) - return stub - - with _Monkey(MUT, make_stub=mock_make_stub): - client.start() - - self.assertTrue(client._data_stub_internal is stub) - if admin: - self.assertTrue(client._instance_stub_internal is stub) - self.assertTrue(client._operations_stub_internal is stub) - self.assertTrue(client._table_stub_internal is stub) - self.assertEqual(stub._entered, 4) - self.assertEqual(len(make_stub_args), 4) - else: - self.assertTrue(client._instance_stub_internal is None) - self.assertTrue(client._operations_stub_internal is None) - self.assertTrue(client._table_stub_internal is None) - self.assertEqual(stub._entered, 1) - self.assertEqual(len(make_stub_args), 1) - self.assertEqual(stub._exited, []) - - def test_start_non_admin(self): - self._start_method_helper(admin=False) - - def test_start_with_admin(self): - self._start_method_helper(admin=True) - - def test_start_while_started(self): - credentials = _Credentials() - project = 'PROJECT' - client = self._makeOne(project=project, credentials=credentials) - client._data_stub_internal = data_stub = object() - self.assertTrue(client.is_started()) - client.start() - - # Make sure the stub did not change. - self.assertEqual(client._data_stub_internal, data_stub) - - def _stop_method_helper(self, admin): - from gcloud.bigtable._testing import _FakeStub - - credentials = _Credentials() - project = 'PROJECT' - client = self._makeOne(project=project, credentials=credentials, - admin=admin) - - stub1 = _FakeStub() - stub2 = _FakeStub() - client._data_stub_internal = stub1 - client._instance_stub_internal = stub2 - client._operations_stub_internal = stub2 - client._table_stub_internal = stub2 - client.stop() - self.assertTrue(client._data_stub_internal is None) - self.assertTrue(client._instance_stub_internal is None) - self.assertTrue(client._operations_stub_internal is None) - self.assertTrue(client._table_stub_internal is None) - self.assertEqual(stub1._entered, 0) - self.assertEqual(stub2._entered, 0) - exc_none_triple = (None, None, None) - self.assertEqual(stub1._exited, [exc_none_triple]) - if admin: - self.assertEqual(stub2._exited, [exc_none_triple] * 3) - else: - self.assertEqual(stub2._exited, []) - - def test_stop_non_admin(self): - self._stop_method_helper(admin=False) - - def test_stop_with_admin(self): - self._stop_method_helper(admin=True) - - def test_stop_while_stopped(self): - credentials = _Credentials() - project = 'PROJECT' - client = self._makeOne(project=project, credentials=credentials) - self.assertFalse(client.is_started()) - - # This is a bit hacky. We set the cluster stub protected value - # since it isn't used in is_started() and make sure that stop - # doesn't reset this value to None. - client._instance_stub_internal = instance_stub = object() - client.stop() - # Make sure the cluster stub did not change. - self.assertEqual(client._instance_stub_internal, instance_stub) - def test_instance_factory_defaults(self): from gcloud.bigtable.cluster import DEFAULT_SERVE_NODES from gcloud.bigtable.instance import Instance @@ -539,7 +415,8 @@ def test_instance_factory_defaults(self): INSTANCE_ID = 'instance-id' DISPLAY_NAME = 'display-name' credentials = _Credentials() - client = self._makeOne(project=PROJECT, credentials=credentials) + client = self._makeOneWithMocks(project=PROJECT, + credentials=credentials) instance = client.instance(INSTANCE_ID, display_name=DISPLAY_NAME) @@ -560,7 +437,8 @@ def test_instance_factory_w_explicit_serve_nodes(self): LOCATION_ID = 'locname' SERVE_NODES = 5 credentials = _Credentials() - client = self._makeOne(project=PROJECT, credentials=credentials) + client = self._makeOneWithMocks(project=PROJECT, + credentials=credentials) instance = client.instance( INSTANCE_ID, display_name=DISPLAY_NAME, @@ -590,11 +468,10 @@ def test_list_instances(self): 'projects/' + self.PROJECT + '/instances/' + INSTANCE_ID2) credentials = _Credentials() - client = self._makeOne( + client = self._makeOneWithMocks( project=self.PROJECT, credentials=credentials, admin=True, - timeout_seconds=self.TIMEOUT_SECONDS, ) # Create request_pb @@ -635,7 +512,7 @@ def test_list_instances(self): self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( 'ListInstances', - (request_pb, self.TIMEOUT_SECONDS), + (request_pb,), {}, )]) @@ -654,3 +531,21 @@ def create_scoped(self, scope): def __eq__(self, other): return self._access_token == other._access_token + + +class _Client(object): + + def __init__(self, credentials, user_agent): + self.credentials = credentials + self.user_agent = user_agent + + +class _MakeStubMock(object): + + def __init__(self): + self.result = object() + self.calls = [] + + def __call__(self, client): + self.calls.append(client) + return self.result diff --git a/gcloud/bigtable/test_cluster.py b/gcloud/bigtable/test_cluster.py index ff21a92833c2..0569f1ea046a 100644 --- a/gcloud/bigtable/test_cluster.py +++ b/gcloud/bigtable/test_cluster.py @@ -85,9 +85,8 @@ def _finished_helper(self, done): CLUSTER_ID = 'cluster-id' OP_TYPE = 'fake-op' OP_ID = 789 - timeout_seconds = 1 - client = _Client(PROJECT, timeout_seconds=timeout_seconds) + client = _Client(PROJECT) instance = _Instance(INSTANCE_ID, client) cluster = Cluster(CLUSTER_ID, instance) operation = self._makeOne(OP_TYPE, OP_ID, cluster=cluster) @@ -114,7 +113,7 @@ def _finished_helper(self, done): self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( 'GetOperation', - (request_pb, timeout_seconds), + (request_pb,), {}, )]) @@ -138,7 +137,6 @@ class TestCluster(unittest.TestCase): CLUSTER_NAME = ('projects/' + PROJECT + '/instances/' + INSTANCE_ID + '/clusters/' + CLUSTER_ID) - TIMEOUT_SECONDS = 123 def _getTargetClass(self): from gcloud.bigtable.cluster import Cluster @@ -307,7 +305,7 @@ def test_reload(self): SERVE_NODES = 31 LOCATION = 'LOCATION' - client = _Client(self.PROJECT, timeout_seconds=self.TIMEOUT_SECONDS) + client = _Client(self.PROJECT) instance = _Instance(self.INSTANCE_ID, client) cluster = self._makeOne(self.CLUSTER_ID, instance) @@ -334,7 +332,7 @@ def test_reload(self): self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( 'GetCluster', - (request_pb, self.TIMEOUT_SECONDS), + (request_pb,), {}, )]) @@ -348,7 +346,7 @@ def test_create(self): from gcloud.bigtable._testing import _FakeStub from gcloud.bigtable import cluster as MUT - client = _Client(self.PROJECT, timeout_seconds=self.TIMEOUT_SECONDS) + client = _Client(self.PROJECT) instance = _Instance(self.INSTANCE_ID, client) cluster = self._makeOne(self.CLUSTER_ID, instance) @@ -390,7 +388,7 @@ def mock_process_operation(operation_pb): self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( 'CreateCluster', - (request_pb, self.TIMEOUT_SECONDS), + (request_pb,), {}, )]) self.assertEqual(prep_create_called, [cluster]) @@ -404,7 +402,7 @@ def test_update(self): SERVE_NODES = 81 - client = _Client(self.PROJECT, timeout_seconds=self.TIMEOUT_SECONDS) + client = _Client(self.PROJECT) instance = _Instance(self.INSTANCE_ID, client) cluster = self._makeOne(self.CLUSTER_ID, instance, serve_nodes=SERVE_NODES) @@ -439,7 +437,7 @@ def mock_process_operation(operation_pb): self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( 'UpdateCluster', - (request_pb, self.TIMEOUT_SECONDS), + (request_pb,), {}, )]) self.assertEqual(process_operation_called, [response_pb]) @@ -448,7 +446,7 @@ def test_delete(self): from google.protobuf import empty_pb2 from gcloud.bigtable._testing import _FakeStub - client = _Client(self.PROJECT, timeout_seconds=self.TIMEOUT_SECONDS) + client = _Client(self.PROJECT) instance = _Instance(self.INSTANCE_ID, client) cluster = self._makeOne(self.CLUSTER_ID, instance) @@ -470,7 +468,7 @@ def test_delete(self): self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( 'DeleteCluster', - (request_pb, self.TIMEOUT_SECONDS), + (request_pb,), {}, )]) @@ -632,12 +630,10 @@ def __eq__(self, other): class _Client(object): - def __init__(self, project, timeout_seconds=None): + def __init__(self, project): self.project = project self.project_name = 'projects/' + self.project - self.timeout_seconds = timeout_seconds def __eq__(self, other): return (other.project == self.project and - other.project_name == self.project_name and - other.timeout_seconds == self.timeout_seconds) + other.project_name == self.project_name) diff --git a/gcloud/bigtable/test_column_family.py b/gcloud/bigtable/test_column_family.py index 8c4b48014a60..207a3df99cb1 100644 --- a/gcloud/bigtable/test_column_family.py +++ b/gcloud/bigtable/test_column_family.py @@ -414,11 +414,10 @@ def _create_test_helper(self, gc_rule=None): cluster_id = 'cluster-id' table_id = 'table-id' column_family_id = 'column-family-id' - timeout_seconds = 4 table_name = ('projects/' + project_id + '/zones/' + zone + '/clusters/' + cluster_id + '/tables/' + table_id) - client = _Client(timeout_seconds=timeout_seconds) + client = _Client() table = _Table(table_name, client=client) column_family = self._makeOne( column_family_id, table, gc_rule=gc_rule) @@ -451,7 +450,7 @@ def _create_test_helper(self, gc_rule=None): self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( 'ModifyColumnFamilies', - (request_pb, timeout_seconds), + (request_pb,), {}, )]) @@ -473,11 +472,10 @@ def _update_test_helper(self, gc_rule=None): cluster_id = 'cluster-id' table_id = 'table-id' column_family_id = 'column-family-id' - timeout_seconds = 28 table_name = ('projects/' + project_id + '/zones/' + zone + '/clusters/' + cluster_id + '/tables/' + table_id) - client = _Client(timeout_seconds=timeout_seconds) + client = _Client() table = _Table(table_name, client=client) column_family = self._makeOne( column_family_id, table, gc_rule=gc_rule) @@ -510,7 +508,7 @@ def _update_test_helper(self, gc_rule=None): self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( 'ModifyColumnFamilies', - (request_pb, timeout_seconds), + (request_pb,), {}, )]) @@ -533,11 +531,10 @@ def test_delete(self): cluster_id = 'cluster-id' table_id = 'table-id' column_family_id = 'column-family-id' - timeout_seconds = 7 table_name = ('projects/' + project_id + '/zones/' + zone + '/clusters/' + cluster_id + '/tables/' + table_id) - client = _Client(timeout_seconds=timeout_seconds) + client = _Client() table = _Table(table_name, client=client) column_family = self._makeOne(column_family_id, table) @@ -564,7 +561,7 @@ def test_delete(self): self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( 'ModifyColumnFamilies', - (request_pb, timeout_seconds), + (request_pb,), {}, )]) @@ -673,9 +670,7 @@ def __init__(self, client=None): class _Client(object): - - def __init__(self, timeout_seconds=None): - self.timeout_seconds = timeout_seconds + pass class _Table(object): diff --git a/gcloud/bigtable/test_instance.py b/gcloud/bigtable/test_instance.py index 950b0c828f11..fd44dfcb4335 100644 --- a/gcloud/bigtable/test_instance.py +++ b/gcloud/bigtable/test_instance.py @@ -94,9 +94,8 @@ def _finished_helper(self, done): PROJECT = 'PROJECT' INSTANCE_ID = 'instance-id' - TIMEOUT_SECONDS = 1 - client = _Client(PROJECT, timeout_seconds=TIMEOUT_SECONDS) + client = _Client(PROJECT) instance = Instance(INSTANCE_ID, client, self.LOCATION_ID) operation = self._makeOne( self.OP_TYPE, self.OP_ID, self.BEGIN, self.LOCATION_ID, @@ -124,7 +123,7 @@ def _finished_helper(self, done): self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( 'GetOperation', - (request_pb, TIMEOUT_SECONDS), + (request_pb,), {}, )]) @@ -153,7 +152,6 @@ class TestInstance(unittest.TestCase): (PROJECT, INSTANCE_ID, OP_ID)) TABLE_ID = 'table_id' TABLE_NAME = INSTANCE_NAME + '/tables/' + TABLE_ID - TIMEOUT_SECONDS = 1 def _getTargetClass(self): from gcloud.bigtable.instance import Instance @@ -316,7 +314,7 @@ def test_reload(self): bigtable_instance_admin_pb2 as messages_v2_pb) from gcloud.bigtable._testing import _FakeStub - client = _Client(self.PROJECT, timeout_seconds=self.TIMEOUT_SECONDS) + client = _Client(self.PROJECT) instance = self._makeOne(self.INSTANCE_ID, client, self.LOCATION_ID) # Create request_pb @@ -343,7 +341,7 @@ def test_reload(self): self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( 'GetInstance', - (request_pb, self.TIMEOUT_SECONDS), + (request_pb,), {}, )]) @@ -356,7 +354,7 @@ def test_create(self): from gcloud.bigtable._testing import _FakeStub from gcloud.bigtable import instance as MUT - client = _Client(self.PROJECT, timeout_seconds=self.TIMEOUT_SECONDS) + client = _Client(self.PROJECT) instance = self._makeOne(self.INSTANCE_ID, client, self.LOCATION_ID) # Create request_pb. Just a mock since we monkey patch @@ -396,7 +394,7 @@ def mock_process_operation(operation_pb): self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( 'CreateInstance', - (request_pb, self.TIMEOUT_SECONDS), + (request_pb,), {}, )]) self.assertEqual(prep_create_called, [instance]) @@ -410,7 +408,7 @@ def test_create_w_explicit_serve_nodes(self): SERVE_NODES = 5 - client = _Client(self.PROJECT, timeout_seconds=self.TIMEOUT_SECONDS) + client = _Client(self.PROJECT) instance = self._makeOne(self.INSTANCE_ID, client, self.LOCATION_ID, serve_nodes=SERVE_NODES) @@ -451,7 +449,7 @@ def mock_process_operation(operation_pb): self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( 'CreateInstance', - (request_pb, self.TIMEOUT_SECONDS), + (request_pb,), {}, )]) self.assertEqual(prep_create_called, [instance]) @@ -462,7 +460,7 @@ def test_update(self): instance_pb2 as data_v2_pb2) from gcloud.bigtable._testing import _FakeStub - client = _Client(self.PROJECT, timeout_seconds=self.TIMEOUT_SECONDS) + client = _Client(self.PROJECT) instance = self._makeOne(self.INSTANCE_ID, client, self.LOCATION_ID, display_name=self.DISPLAY_NAME) @@ -487,7 +485,7 @@ def test_update(self): self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( 'UpdateInstance', - (request_pb, self.TIMEOUT_SECONDS), + (request_pb,), {}, )]) @@ -497,7 +495,7 @@ def test_delete(self): bigtable_instance_admin_pb2 as messages_v2_pb) from gcloud.bigtable._testing import _FakeStub - client = _Client(self.PROJECT, timeout_seconds=self.TIMEOUT_SECONDS) + client = _Client(self.PROJECT) instance = self._makeOne(self.INSTANCE_ID, client, self.LOCATION_ID) # Create request_pb @@ -519,7 +517,7 @@ def test_delete(self): self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( 'DeleteInstance', - (request_pb, self.TIMEOUT_SECONDS), + (request_pb,), {}, )]) @@ -536,7 +534,7 @@ def test_list_clusters(self): CLUSTER_ID2 = 'cluster-id2' SERVE_NODES = 4 - client = _Client(self.PROJECT, timeout_seconds=self.TIMEOUT_SECONDS) + client = _Client(self.PROJECT) instance = self._makeOne(self.INSTANCE_ID, client, self.LOCATION_ID) CLUSTER_NAME1 = (instance.name + '/clusters/' + CLUSTER_ID1) @@ -576,7 +574,7 @@ def test_list_clusters(self): self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( 'ListClusters', - (request_pb, self.TIMEOUT_SECONDS), + (request_pb,), {}, )]) @@ -587,7 +585,7 @@ def _list_tables_helper(self, table_name=None): bigtable_table_admin_pb2 as table_messages_v1_pb2) from gcloud.bigtable._testing import _FakeStub - client = _Client(self.PROJECT, timeout_seconds=self.TIMEOUT_SECONDS) + client = _Client(self.PROJECT) instance = self._makeOne(self.INSTANCE_ID, client, self.LOCATION_ID) # Create request_ @@ -617,7 +615,7 @@ def _list_tables_helper(self, table_name=None): self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( 'ListTables', - (request_pb, self.TIMEOUT_SECONDS), + (request_pb,), {}, )]) @@ -851,10 +849,9 @@ def test_op_name_parsing_failure(self): class _Client(object): - def __init__(self, project, timeout_seconds=None): + def __init__(self, project): self.project = project self.project_name = 'projects/' + self.project - self.timeout_seconds = timeout_seconds def copy(self): from copy import deepcopy @@ -862,5 +859,4 @@ def copy(self): def __eq__(self, other): return (other.project == self.project and - other.project_name == self.project_name and - other.timeout_seconds == self.timeout_seconds) + other.project_name == self.project_name) diff --git a/gcloud/bigtable/test_row.py b/gcloud/bigtable/test_row.py index 6c9e52fc6c78..8331a68fcb73 100644 --- a/gcloud/bigtable/test_row.py +++ b/gcloud/bigtable/test_row.py @@ -302,8 +302,7 @@ def test_commit(self): table_name = 'projects/more-stuff' column_family_id = u'column_family_id' column = b'column' - timeout_seconds = 711 - client = _Client(timeout_seconds=timeout_seconds) + client = _Client() table = _Table(table_name, client=client) row = self._makeOne(row_key, table) @@ -338,7 +337,7 @@ def test_commit(self): self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( 'MutateRow', - (request_pb, timeout_seconds), + (request_pb,), {}, )]) self.assertEqual(row._pb_mutations, []) @@ -418,8 +417,7 @@ def test_commit(self): column_family_id3 = u'column_family_id3' column1 = b'column1' column2 = b'column2' - timeout_seconds = 262 - client = _Client(timeout_seconds=timeout_seconds) + client = _Client() table = _Table(table_name, client=client) row_filter = RowSampleFilter(0.33) row = self._makeOne(row_key, table, filter_=row_filter) @@ -476,7 +474,7 @@ def test_commit(self): self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( 'CheckAndMutateRow', - (request_pb, timeout_seconds), + (request_pb,), {}, )]) self.assertEqual(row._true_pb_mutations, []) @@ -582,8 +580,7 @@ def test_commit(self): table_name = 'projects/more-stuff' column_family_id = u'column_family_id' column = b'column' - timeout_seconds = 87 - client = _Client(timeout_seconds=timeout_seconds) + client = _Client() table = _Table(table_name, client=client) row = self._makeOne(row_key, table) @@ -624,7 +621,7 @@ def mock_parse_rmw_row_response(row_response): self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( 'ReadModifyWriteRow', - (request_pb, timeout_seconds), + (request_pb,), {}, )]) self.assertEqual(row_responses, [response_pb]) @@ -898,9 +895,6 @@ class _Client(object): data_stub = None - def __init__(self, timeout_seconds=None): - self.timeout_seconds = timeout_seconds - class _Instance(object): diff --git a/gcloud/bigtable/test_row_data.py b/gcloud/bigtable/test_row_data.py index 28b9971ea667..ecff8dc2bd9d 100644 --- a/gcloud/bigtable/test_row_data.py +++ b/gcloud/bigtable/test_row_data.py @@ -713,7 +713,10 @@ def _parse_readrows_acceptance_tests(filename): """Parse acceptance tests from JSON See: - https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/master/bigtable-client-core/src/test/resources/com/google/cloud/bigtable/grpc/scanner/v2/read-rows-acceptance-test.json + https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/\ + 4d3185662ca61bc9fa1bdf1ec0166f6e5ecf86c6/bigtable-client-core/src/\ + test/resources/com/google/cloud/bigtable/grpc/scanner/v2/ + read-rows-acceptance-test.json """ import json diff --git a/gcloud/bigtable/test_table.py b/gcloud/bigtable/test_table.py index 0b038623d6b8..4d7a25d50d46 100644 --- a/gcloud/bigtable/test_table.py +++ b/gcloud/bigtable/test_table.py @@ -23,7 +23,6 @@ class TestTable(unittest.TestCase): INSTANCE_NAME = ('projects/' + PROJECT_ID + '/instances/' + INSTANCE_ID) TABLE_ID = 'table-id' TABLE_NAME = INSTANCE_NAME + '/tables/' + TABLE_ID - TIMEOUT_SECONDS = 1333 ROW_KEY = b'row-key' FAMILY_NAME = u'family' QUALIFIER = b'qualifier' @@ -137,7 +136,7 @@ def _create_test_helper(self, initial_split_keys, column_families=()): from gcloud._helpers import _to_bytes from gcloud.bigtable._testing import _FakeStub - client = _Client(timeout_seconds=self.TIMEOUT_SECONDS) + client = _Client() instance = _Instance(self.INSTANCE_NAME, client=client) table = self._makeOne(self.TABLE_ID, instance) @@ -174,7 +173,7 @@ def _create_test_helper(self, initial_split_keys, column_families=()): self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( 'CreateTable', - (request_pb, self.TIMEOUT_SECONDS), + (request_pb,), {}, )]) @@ -204,7 +203,7 @@ def test_create_with_column_families(self): def _list_column_families_helper(self): from gcloud.bigtable._testing import _FakeStub - client = _Client(timeout_seconds=self.TIMEOUT_SECONDS) + client = _Client() instance = _Instance(self.INSTANCE_NAME, client=client) table = self._makeOne(self.TABLE_ID, instance) @@ -231,7 +230,7 @@ def _list_column_families_helper(self): self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( 'GetTable', - (request_pb, self.TIMEOUT_SECONDS), + (request_pb,), {}, )]) @@ -242,7 +241,7 @@ def test_delete(self): from google.protobuf import empty_pb2 from gcloud.bigtable._testing import _FakeStub - client = _Client(timeout_seconds=self.TIMEOUT_SECONDS) + client = _Client() instance = _Instance(self.INSTANCE_NAME, client=client) table = self._makeOne(self.TABLE_ID, instance) @@ -263,7 +262,7 @@ def test_delete(self): self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( 'DeleteTable', - (request_pb, self.TIMEOUT_SECONDS), + (request_pb,), {}, )]) @@ -272,7 +271,7 @@ def _read_row_helper(self, chunks, expected_result): from gcloud.bigtable._testing import _FakeStub from gcloud.bigtable import table as MUT - client = _Client(timeout_seconds=self.TIMEOUT_SECONDS) + client = _Client() instance = _Instance(self.INSTANCE_NAME, client=client) table = self._makeOne(self.TABLE_ID, instance) @@ -302,7 +301,7 @@ def mock_create_row_request(table_name, row_key, filter_): self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( 'ReadRows', - (request_pb, self.TIMEOUT_SECONDS), + (request_pb,), {}, )]) self.assertEqual(mock_created, @@ -353,7 +352,7 @@ def test_read_rows(self): from gcloud.bigtable.row_data import PartialRowsData from gcloud.bigtable import table as MUT - client = _Client(timeout_seconds=self.TIMEOUT_SECONDS) + client = _Client() instance = _Instance(self.INSTANCE_NAME, client=client) table = self._makeOne(self.TABLE_ID, instance) @@ -387,7 +386,7 @@ def mock_create_row_request(table_name, **kwargs): self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( 'ReadRows', - (request_pb, self.TIMEOUT_SECONDS), + (request_pb,), {}, )]) created_kwargs = { @@ -401,7 +400,7 @@ def mock_create_row_request(table_name, **kwargs): def test_sample_row_keys(self): from gcloud.bigtable._testing import _FakeStub - client = _Client(timeout_seconds=self.TIMEOUT_SECONDS) + client = _Client() instance = _Instance(self.INSTANCE_NAME, client=client) table = self._makeOne(self.TABLE_ID, instance) @@ -422,7 +421,7 @@ def test_sample_row_keys(self): self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( 'SampleRowKeys', - (request_pb, self.TIMEOUT_SECONDS), + (request_pb,), {}, )]) @@ -578,9 +577,6 @@ class _Client(object): operations_stub = None table_stub = None - def __init__(self, timeout_seconds=None): - self.timeout_seconds = timeout_seconds - class _Instance(object): diff --git a/gcloud/datastore/connection.py b/gcloud/datastore/connection.py index 6df414acd930..d1fc241be67d 100644 --- a/gcloud/datastore/connection.py +++ b/gcloud/datastore/connection.py @@ -26,15 +26,14 @@ from gcloud.datastore._generated import datastore_pb2 as _datastore_pb2 # pylint: disable=ungrouped-imports try: - from grpc.beta.interfaces import StatusCode - from grpc.framework.interfaces.face.face import AbortionError + from grpc import StatusCode + from grpc._channel import _Rendezvous from gcloud.datastore._generated import datastore_grpc_pb2 - DATASTORE_STUB_FACTORY = datastore_grpc_pb2.beta_create_Datastore_stub except ImportError: # pragma: NO COVER _HAVE_GRPC = False - DATASTORE_STUB_FACTORY = None + datastore_grpc_pb2 = None StatusCode = None - AbortionError = Exception + _Rendezvous = Exception else: _HAVE_GRPC = True # pylint: enable=ungrouped-imports @@ -44,8 +43,6 @@ """Datastore API request host.""" DATASTORE_API_PORT = 443 """Datastore API request port.""" -GRPC_TIMEOUT_SECONDS = 10 -"""The default timeout to use for API requests via gRPC.""" class _DatastoreAPIOverHttp(object): @@ -237,20 +234,8 @@ class _DatastoreAPIOverGRPC(object): def __init__(self, connection): self._stub = make_stub(connection.credentials, connection.USER_AGENT, - DATASTORE_STUB_FACTORY, DATASTORE_API_HOST, - DATASTORE_API_PORT) - self._stub.__enter__() - - def __del__(self): - """Destructor for object. - - Ensures that the stub is exited so the shell can close properly. - """ - try: - self._stub.__exit__(None, None, None) - del self._stub - except AttributeError: - pass + datastore_grpc_pb2.DatastoreStub, + DATASTORE_API_HOST, DATASTORE_API_PORT) def lookup(self, project, request_pb): """Perform a ``lookup`` request. @@ -266,7 +251,7 @@ def lookup(self, project, request_pb): :returns: The returned protobuf response object. """ request_pb.project_id = project - return self._stub.Lookup(request_pb, GRPC_TIMEOUT_SECONDS) + return self._stub.Lookup(request_pb) def run_query(self, project, request_pb): """Perform a ``runQuery`` request. @@ -282,7 +267,7 @@ def run_query(self, project, request_pb): :returns: The returned protobuf response object. """ request_pb.project_id = project - return self._stub.RunQuery(request_pb, GRPC_TIMEOUT_SECONDS) + return self._stub.RunQuery(request_pb) def begin_transaction(self, project, request_pb): """Perform a ``beginTransaction`` request. @@ -299,7 +284,7 @@ def begin_transaction(self, project, request_pb): :returns: The returned protobuf response object. """ request_pb.project_id = project - return self._stub.BeginTransaction(request_pb, GRPC_TIMEOUT_SECONDS) + return self._stub.BeginTransaction(request_pb) def commit(self, project, request_pb): """Perform a ``commit`` request. @@ -316,10 +301,10 @@ def commit(self, project, request_pb): """ request_pb.project_id = project try: - return self._stub.Commit(request_pb, GRPC_TIMEOUT_SECONDS) - except AbortionError as exc: - if exc.code == StatusCode.ABORTED: - raise Conflict(exc.details) + return self._stub.Commit(request_pb) + except _Rendezvous as exc: + if exc.code() == StatusCode.ABORTED: + raise Conflict(exc.details()) raise def rollback(self, project, request_pb): @@ -336,7 +321,7 @@ def rollback(self, project, request_pb): :returns: The returned protobuf response object. """ request_pb.project_id = project - return self._stub.Rollback(request_pb, GRPC_TIMEOUT_SECONDS) + return self._stub.Rollback(request_pb) def allocate_ids(self, project, request_pb): """Perform an ``allocateIds`` request. @@ -352,7 +337,7 @@ def allocate_ids(self, project, request_pb): :returns: The returned protobuf response object. """ request_pb.project_id = project - return self._stub.AllocateIds(request_pb, GRPC_TIMEOUT_SECONDS) + return self._stub.AllocateIds(request_pb) class Connection(connection_module.Connection): diff --git a/gcloud/datastore/test_connection.py b/gcloud/datastore/test_connection.py index 3ed80f18bedf..cfa0dc80d906 100644 --- a/gcloud/datastore/test_connection.py +++ b/gcloud/datastore/test_connection.py @@ -112,7 +112,7 @@ def _getTargetClass(self): from gcloud.datastore.connection import _DatastoreAPIOverGRPC return _DatastoreAPIOverGRPC - def _makeOne(self, connection=None, stub=None, mock_args=None): + def _makeOne(self, stub, connection=None, mock_args=None): from gcloud._testing import _Monkey from gcloud.datastore import connection as MUT @@ -120,9 +120,6 @@ def _makeOne(self, connection=None, stub=None, mock_args=None): connection = _Connection(None) connection.credentials = object() - if stub is None: - stub = _GRPCStub() - if mock_args is None: mock_args = [] @@ -141,43 +138,19 @@ def test_constructor(self): stub = _GRPCStub() mock_args = [] - self.assertEqual(stub.enter_calls, 0) - datastore_api = self._makeOne(conn, stub=stub, mock_args=mock_args) + datastore_api = self._makeOne(stub, connection=conn, + mock_args=mock_args) self.assertIs(datastore_api._stub, stub) self.assertEqual(mock_args, [( conn.credentials, conn.USER_AGENT, - MUT.DATASTORE_STUB_FACTORY, + MUT.datastore_grpc_pb2.DatastoreStub, MUT.DATASTORE_API_HOST, MUT.DATASTORE_API_PORT, )]) - def test___del__valid_stub(self): - datastore_api = self._makeOne() - - stub = datastore_api._stub - self.assertEqual(stub.exit_calls, []) - self.assertIs(datastore_api._stub, stub) - datastore_api.__del__() - self.assertEqual(stub.exit_calls, [(None, None, None)]) - self.assertFalse(hasattr(datastore_api, '_stub')) - - def test___del__invalid_stub(self): - datastore_api = self._makeOne() - - stub = datastore_api._stub - self.assertEqual(stub.exit_calls, []) - - del datastore_api._stub - self.assertFalse(hasattr(datastore_api, '_stub')) - datastore_api.__del__() - self.assertEqual(stub.exit_calls, []) - self.assertFalse(hasattr(datastore_api, '_stub')) - def test_lookup(self): - from gcloud.datastore import connection as MUT - return_val = object() stub = _GRPCStub(return_val) datastore_api = self._makeOne(stub=stub) @@ -188,11 +161,9 @@ def test_lookup(self): self.assertIs(result, return_val) self.assertEqual(request_pb.project_id, project) self.assertEqual(stub.method_calls, - [(request_pb, MUT.GRPC_TIMEOUT_SECONDS, 'Lookup')]) + [(request_pb, 'Lookup')]) def test_run_query(self): - from gcloud.datastore import connection as MUT - return_val = object() stub = _GRPCStub(return_val) datastore_api = self._makeOne(stub=stub) @@ -203,11 +174,9 @@ def test_run_query(self): self.assertIs(result, return_val) self.assertEqual(request_pb.project_id, project) self.assertEqual(stub.method_calls, - [(request_pb, MUT.GRPC_TIMEOUT_SECONDS, 'RunQuery')]) + [(request_pb, 'RunQuery')]) def test_begin_transaction(self): - from gcloud.datastore import connection as MUT - return_val = object() stub = _GRPCStub(return_val) datastore_api = self._makeOne(stub=stub) @@ -219,11 +188,9 @@ def test_begin_transaction(self): self.assertEqual(request_pb.project_id, project) self.assertEqual( stub.method_calls, - [(request_pb, MUT.GRPC_TIMEOUT_SECONDS, 'BeginTransaction')]) + [(request_pb, 'BeginTransaction')]) def test_commit_success(self): - from gcloud.datastore import connection as MUT - return_val = object() stub = _GRPCStub(return_val) datastore_api = self._makeOne(stub=stub) @@ -234,11 +201,9 @@ def test_commit_success(self): self.assertIs(result, return_val) self.assertEqual(request_pb.project_id, project) self.assertEqual(stub.method_calls, - [(request_pb, MUT.GRPC_TIMEOUT_SECONDS, 'Commit')]) + [(request_pb, 'Commit')]) def _commit_failure_helper(self, exc, err_class): - from gcloud.datastore import connection as MUT - stub = _GRPCStub(side_effect=exc) datastore_api = self._makeOne(stub=stub) @@ -249,24 +214,29 @@ def _commit_failure_helper(self, exc, err_class): self.assertEqual(request_pb.project_id, project) self.assertEqual(stub.method_calls, - [(request_pb, MUT.GRPC_TIMEOUT_SECONDS, 'Commit')]) + [(request_pb, 'Commit')]) @unittest.skipUnless(_HAVE_GRPC, 'No gRPC') def test_commit_failure_aborted(self): - from grpc.beta.interfaces import StatusCode - from grpc.framework.interfaces.face.face import AbortionError + from grpc import StatusCode + from grpc._channel import _Rendezvous + from grpc._channel import _RPCState from gcloud.exceptions import Conflict - exc = AbortionError(None, None, StatusCode.ABORTED, None) + details = 'Bad things.' + exc_state = _RPCState((), None, None, StatusCode.ABORTED, details) + exc = _Rendezvous(exc_state, None, None, None) self._commit_failure_helper(exc, Conflict) @unittest.skipUnless(_HAVE_GRPC, 'No gRPC') def test_commit_failure_cancelled(self): - from grpc.beta.interfaces import StatusCode - from grpc.framework.interfaces.face.face import AbortionError + from grpc import StatusCode + from grpc._channel import _Rendezvous + from grpc._channel import _RPCState - exc = AbortionError(None, None, StatusCode.CANCELLED, None) - self._commit_failure_helper(exc, AbortionError) + exc_state = _RPCState((), None, None, StatusCode.CANCELLED, None) + exc = _Rendezvous(exc_state, None, None, None) + self._commit_failure_helper(exc, _Rendezvous) @unittest.skipUnless(_HAVE_GRPC, 'No gRPC') def test_commit_failure_non_grpc_err(self): @@ -274,8 +244,6 @@ def test_commit_failure_non_grpc_err(self): self._commit_failure_helper(exc, RuntimeError) def test_rollback(self): - from gcloud.datastore import connection as MUT - return_val = object() stub = _GRPCStub(return_val) datastore_api = self._makeOne(stub=stub) @@ -286,11 +254,9 @@ def test_rollback(self): self.assertIs(result, return_val) self.assertEqual(request_pb.project_id, project) self.assertEqual(stub.method_calls, - [(request_pb, MUT.GRPC_TIMEOUT_SECONDS, 'Rollback')]) + [(request_pb, 'Rollback')]) def test_allocate_ids(self): - from gcloud.datastore import connection as MUT - return_val = object() stub = _GRPCStub(return_val) datastore_api = self._makeOne(stub=stub) @@ -302,7 +268,7 @@ def test_allocate_ids(self): self.assertEqual(request_pb.project_id, project) self.assertEqual( stub.method_calls, - [(request_pb, MUT.GRPC_TIMEOUT_SECONDS, 'AllocateIds')]) + [(request_pb, 'AllocateIds')]) class TestConnection(unittest.TestCase): @@ -1155,42 +1121,33 @@ class _GRPCStub(object): def __init__(self, return_val=None, side_effect=Exception): self.return_val = return_val self.side_effect = side_effect - self.enter_calls = 0 - self.exit_calls = [] self.method_calls = [] - def __enter__(self): - self.enter_calls += 1 - return self - - def __exit__(self, *args): - self.exit_calls.append(args) - - def _method(self, request_pb, timeout, name): - self.method_calls.append((request_pb, timeout, name)) + def _method(self, request_pb, name): + self.method_calls.append((request_pb, name)) return self.return_val - def Lookup(self, request_pb, timeout): - return self._method(request_pb, timeout, 'Lookup') + def Lookup(self, request_pb): + return self._method(request_pb, 'Lookup') - def RunQuery(self, request_pb, timeout): - return self._method(request_pb, timeout, 'RunQuery') + def RunQuery(self, request_pb): + return self._method(request_pb, 'RunQuery') - def BeginTransaction(self, request_pb, timeout): - return self._method(request_pb, timeout, 'BeginTransaction') + def BeginTransaction(self, request_pb): + return self._method(request_pb, 'BeginTransaction') - def Commit(self, request_pb, timeout): - result = self._method(request_pb, timeout, 'Commit') + def Commit(self, request_pb): + result = self._method(request_pb, 'Commit') if self.side_effect is Exception: return result else: raise self.side_effect - def Rollback(self, request_pb, timeout): - return self._method(request_pb, timeout, 'Rollback') + def Rollback(self, request_pb): + return self._method(request_pb, 'Rollback') - def AllocateIds(self, request_pb, timeout): - return self._method(request_pb, timeout, 'AllocateIds') + def AllocateIds(self, request_pb): + return self._method(request_pb, 'AllocateIds') class _RequestPB(object): diff --git a/gcloud/logging/_gax.py b/gcloud/logging/_gax.py index fd8c3eff3e2b..26a6a0bbe513 100644 --- a/gcloud/logging/_gax.py +++ b/gcloud/logging/_gax.py @@ -19,18 +19,18 @@ from google.gax import CallOptions from google.gax import INITIAL_PAGE from google.gax.errors import GaxError -from google.gax.grpc import exc_to_code from google.logging.type.log_severity_pb2 import LogSeverity from google.logging.v2.logging_config_pb2 import LogSink from google.logging.v2.logging_metrics_pb2 import LogMetric from google.logging.v2.log_entry_pb2 import LogEntry from google.protobuf.json_format import Parse -from grpc.beta.interfaces import StatusCode +from grpc import StatusCode -from gcloud.exceptions import Conflict -from gcloud.exceptions import NotFound from gcloud._helpers import _datetime_to_pb_timestamp from gcloud._helpers import _pb_timestamp_to_rfc3339 +from gcloud._helpers import exc_to_code +from gcloud.exceptions import Conflict +from gcloud.exceptions import NotFound class _LoggingAPI(object): diff --git a/gcloud/logging/test__gax.py b/gcloud/logging/test__gax.py index bb26ecbc19fc..342b1ac6548d 100644 --- a/gcloud/logging/test__gax.py +++ b/gcloud/logging/test__gax.py @@ -17,7 +17,7 @@ try: # pylint: disable=unused-import - import gcloud.pubsub._gax + import gcloud.logging._gax # pylint: enable=unused-import except ImportError: # pragma: NO COVER _HAVE_GAX = False diff --git a/gcloud/pubsub/_gax.py b/gcloud/pubsub/_gax.py index 4fde46a3f931..68928a0fbe9e 100644 --- a/gcloud/pubsub/_gax.py +++ b/gcloud/pubsub/_gax.py @@ -17,14 +17,14 @@ from google.gax import CallOptions from google.gax import INITIAL_PAGE from google.gax.errors import GaxError -from google.gax.grpc import exc_to_code from google.pubsub.v1.pubsub_pb2 import PubsubMessage from google.pubsub.v1.pubsub_pb2 import PushConfig -from grpc.beta.interfaces import StatusCode +from grpc import StatusCode +from gcloud._helpers import _to_bytes +from gcloud._helpers import exc_to_code from gcloud.exceptions import Conflict from gcloud.exceptions import NotFound -from gcloud._helpers import _to_bytes class _PublisherAPI(object): diff --git a/gcloud/test__helpers.py b/gcloud/test__helpers.py index ea4328567e90..9d5b9542d439 100644 --- a/gcloud/test__helpers.py +++ b/gcloud/test__helpers.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import unittest @@ -155,17 +156,14 @@ def _callFUT(self): return _file_project_id() def setUp(self): - import os self.old_env = os.environ.get('GOOGLE_APPLICATION_CREDENTIALS') def tearDown(self): - import os if (not self.old_env and 'GOOGLE_APPLICATION_CREDENTIALS' in os.environ): del os.environ['GOOGLE_APPLICATION_CREDENTIALS'] def test_success(self): - import os from gcloud._testing import _NamedTemporaryFile with _NamedTemporaryFile() as temp: @@ -181,13 +179,12 @@ def test_no_environment(self): class Test__get_default_service_project_id(unittest.TestCase): - config_path = '.config/gcloud/configurations/' + config_path = os.path.join('.config', 'gcloud', 'configurations') config_file = 'config_default' temp_APPDATA = '' def setUp(self): import tempfile - import os self.temp_config_path = tempfile.mkdtemp() self.temp_APPDATA = os.getenv('APPDATA') @@ -205,14 +202,12 @@ def setUp(self): def tearDown(self): import shutil - import os if os.path.exists(self.temp_config_path): shutil.rmtree(self.temp_config_path) if self.temp_APPDATA: # pragma: NO COVER Windows os.environ['APPDATA'] = self.temp_APPDATA - def callFUT(self, project_id=None): - import os + def _callFUT(self, project_id=None): from gcloud._helpers import _default_service_project_id from gcloud._testing import _Monkey @@ -226,7 +221,7 @@ def mock_expanduser(path=None): return _default_service_project_id() def test_read_from_cli_info(self): - project_id = self.callFUT('test-project-id') + project_id = self._callFUT('test-project-id') self.assertEqual('test-project-id', project_id) def test_gae_without_expanduser(self): @@ -236,7 +231,7 @@ def test_gae_without_expanduser(self): try: sys.modules['pwd'] = None # Blocks pwd from being imported. - project_id = self.callFUT('test-project-id') + project_id = self._callFUT('test-project-id') self.assertEqual(None, project_id) finally: del sys.modules['pwd'] # Unblocks importing of pwd. @@ -244,7 +239,7 @@ def test_gae_without_expanduser(self): def test_info_value_not_present(self): import shutil shutil.rmtree(self.temp_config_path) - project_id = self.callFUT() + project_id = self._callFUT() self.assertEqual(None, project_id) @@ -258,12 +253,12 @@ def _monkeyConnection(self, connection): from gcloud._testing import _Monkey from gcloud import _helpers - def _factory(host, timeout): + def _connection_factory(host, timeout): connection.host = host connection.timeout = timeout return connection - return _Monkey(_helpers, HTTPConnection=_factory) + return _Monkey(_helpers, HTTPConnection=_connection_factory) def test_bad_status(self): connection = _HTTPConnection(404, None) @@ -292,7 +287,6 @@ def _callFUT(self): return _get_production_project() def test_no_value(self): - import os from gcloud._testing import _Monkey environ = {} @@ -301,7 +295,6 @@ def test_no_value(self): self.assertEqual(project, None) def test_value_set(self): - import os from gcloud._testing import _Monkey from gcloud._helpers import PROJECT @@ -871,7 +864,7 @@ def test_it(self): COMPOSITE_CREDS = object() CHANNEL = object() - class _ImplementationsModule(object): + class _GRPCModule(object): def __init__(self): self.ssl_channel_credentials_args = None @@ -895,9 +888,9 @@ def secure_channel(self, *args): self.secure_channel_args = args return CHANNEL - implementations_mod = _ImplementationsModule() + grpc_mod = _GRPCModule() - def mock_stub_factory(channel): + def mock_stub_class(channel): stub_inputs.append(channel) return mock_result @@ -912,23 +905,54 @@ def mock_plugin(*args): port = 1025 credentials = object() user_agent = 'USER_AGENT' - with _Monkey(MUT, implementations=implementations_mod, + with _Monkey(MUT, grpc=grpc_mod, MetadataPlugin=mock_plugin): result = self._callFUT(credentials, user_agent, - mock_stub_factory, host, port) + mock_stub_class, host, port) self.assertTrue(result is mock_result) self.assertEqual(stub_inputs, [CHANNEL]) self.assertEqual(plugin_args, [(credentials, user_agent)]) - self.assertEqual(implementations_mod.ssl_channel_credentials_args, - (None, None, None)) - self.assertEqual(implementations_mod.metadata_call_credentials_args, + self.assertEqual(grpc_mod.ssl_channel_credentials_args, ()) + self.assertEqual(grpc_mod.metadata_call_credentials_args, ((metadata_plugin,), {'name': 'google_creds'})) self.assertEqual( - implementations_mod.composite_channel_credentials_args, + grpc_mod.composite_channel_credentials_args, (SSL_CREDS, METADATA_CREDS)) - self.assertEqual(implementations_mod.secure_channel_args, - (host, port, COMPOSITE_CREDS)) + target = '%s:%d' % (host, port) + self.assertEqual(grpc_mod.secure_channel_args, + (target, COMPOSITE_CREDS)) + + +class Test_exc_to_code(unittest.TestCase): + + def _callFUT(self, exc): + from gcloud._helpers import exc_to_code + return exc_to_code(exc) + + def test_with_stable(self): + from grpc._channel import _Rendezvous + from grpc._channel import _RPCState + from grpc import StatusCode + + status_code = StatusCode.FAILED_PRECONDITION + exc_state = _RPCState((), None, None, status_code, None) + exc = _Rendezvous(exc_state, None, None, None) + result = self._callFUT(exc) + self.assertEqual(result, status_code) + + def test_with_beta(self): + from grpc import StatusCode + from grpc.framework.interfaces.face.face import AbortionError + + status_code = StatusCode.UNIMPLEMENTED + exc = AbortionError(None, None, status_code, None) + result = self._callFUT(exc) + self.assertEqual(result, status_code) + + def test_with_none(self): + result = self._callFUT(None) + self.assertIsNone(result) class _AppIdentity(object): diff --git a/scripts/make_datastore_grpc.py b/scripts/make_datastore_grpc.py index 6e296bb9d72c..7479b77b17a3 100644 --- a/scripts/make_datastore_grpc.py +++ b/scripts/make_datastore_grpc.py @@ -28,7 +28,11 @@ 'v1', 'datastore.proto') GRPC_ONLY_FILE = os.path.join(ROOT_DIR, 'gcloud', 'datastore', '_generated', 'datastore_grpc_pb2.py') -GRPCIO_VIRTUALENV = os.environ.get('GRPCIO_VIRTUALENV', 'protoc') +GRPCIO_VIRTUALENV = os.getenv('GRPCIO_VIRTUALENV') +if GRPCIO_VIRTUALENV is None: + PYTHON_EXECUTABLE = sys.executable +else: + PYTHON_EXECUTABLE = os.path.join(GRPCIO_VIRTUALENV, 'bin', 'python') MESSAGE_SNIPPET = ' = _reflection.GeneratedProtocolMessageType(' IMPORT_TEMPLATE = 'from gcloud.datastore._generated.datastore_pb2 import %s\n' @@ -44,7 +48,7 @@ def get_pb2_contents_with_grpc(): 'v1', 'datastore_pb2.py') try: return_code = subprocess.call([ - '%s/bin/python' % GRPCIO_VIRTUALENV, + PYTHON_EXECUTABLE, '-m', 'grpc.tools.protoc', '--proto_path', @@ -74,7 +78,7 @@ def get_pb2_contents_without_grpc(): 'v1', 'datastore_pb2.py') try: return_code = subprocess.call([ - '%s/bin/python' % GRPCIO_VIRTUALENV, + PYTHON_EXECUTABLE, '-m', 'grpc.tools.protoc', '--proto_path', diff --git a/scripts/make_operations_grpc.py b/scripts/make_operations_grpc.py index 0e779964f3b0..b60f57833386 100644 --- a/scripts/make_operations_grpc.py +++ b/scripts/make_operations_grpc.py @@ -26,10 +26,14 @@ PROTOS_DIR = os.path.join(ROOT_DIR, 'googleapis-pb') PROTO_PATH = os.path.join(PROTOS_DIR, 'google', 'longrunning', 'operations.proto') -GENERATED_SUBDIR = os.environ.get('GENERATED_SUBDIR', '_generated') +GENERATED_SUBDIR = os.getenv('GENERATED_SUBDIR', '_generated') GRPC_ONLY_FILE = os.path.join(ROOT_DIR, 'gcloud', 'bigtable', GENERATED_SUBDIR, 'operations_grpc_pb2.py') -GRPCIO_VIRTUALENV = os.environ.get('GRPCIO_VIRTUALENV', 'protoc') +GRPCIO_VIRTUALENV = os.getenv('GRPCIO_VIRTUALENV') +if GRPCIO_VIRTUALENV is None: + PYTHON_EXECUTABLE = sys.executable +else: + PYTHON_EXECUTABLE = os.path.join(GRPCIO_VIRTUALENV, 'bin', 'python') def get_pb2_contents_with_grpc(): @@ -43,7 +47,7 @@ def get_pb2_contents_with_grpc(): 'operations_pb2.py') try: return_code = subprocess.call([ - '%s/bin/python' % GRPCIO_VIRTUALENV, + PYTHON_EXECUTABLE, '-m', 'grpc.tools.protoc', '--proto_path', @@ -73,7 +77,7 @@ def get_pb2_contents_without_grpc(): 'operations_pb2.py') try: return_code = subprocess.call([ - '%s/bin/python' % GRPCIO_VIRTUALENV, + PYTHON_EXECUTABLE, '-m', 'grpc.tools.protoc', '--proto_path', diff --git a/system_tests/bigtable.py b/system_tests/bigtable.py index 943625610b0b..c7113d4a883f 100644 --- a/system_tests/bigtable.py +++ b/system_tests/bigtable.py @@ -85,18 +85,17 @@ def _operation_finished(result): def _retry_on_unavailable(exc): - """Retry only AbortionErrors whose status code is 'UNAVAILABLE'.""" - from grpc.beta.interfaces import StatusCode - return exc.code == StatusCode.UNAVAILABLE + """Retry only errors whose status code is 'UNAVAILABLE'.""" + from grpc import StatusCode + return exc.code() == StatusCode.UNAVAILABLE def setUpModule(): - from grpc.framework.interfaces.face.face import AbortionError + from grpc._channel import _Rendezvous _helpers.PROJECT = TESTS_PROJECT Config.CLIENT = Client(admin=True) Config.INSTANCE = Config.CLIENT.instance(INSTANCE_ID, LOCATION_ID) - Config.CLIENT.start() - retry = RetryErrors(AbortionError, error_predicate=_retry_on_unavailable) + retry = RetryErrors(_Rendezvous, error_predicate=_retry_on_unavailable) instances, failed_locations = retry(Config.CLIENT.list_instances)() if len(failed_locations) != 0: @@ -112,7 +111,6 @@ def setUpModule(): def tearDownModule(): Config.INSTANCE.delete() - Config.CLIENT.stop() class TestInstanceAdminAPI(unittest.TestCase): diff --git a/system_tests/logging_.py b/system_tests/logging_.py index 6f95d01c0b64..4ef2c6ffc944 100644 --- a/system_tests/logging_.py +++ b/system_tests/logging_.py @@ -34,9 +34,9 @@ def _retry_on_unavailable(exc): - """Retry only AbortionErrors whose status code is 'UNAVAILABLE'.""" - from grpc.beta.interfaces import StatusCode - return exc.code == StatusCode.UNAVAILABLE + """Retry only errors whose status code is 'UNAVAILABLE'.""" + from grpc import StatusCode + return exc.code() == StatusCode.UNAVAILABLE def _has_entries(result): @@ -73,9 +73,9 @@ def _logger_name(): return 'system-tests-logger' + unique_resource_id('-') def _list_entries(self, logger): - from grpc.framework.interfaces.face.face import AbortionError + from grpc._channel import _Rendezvous inner = RetryResult(_has_entries)(logger.list_entries) - outer = RetryErrors(AbortionError, _retry_on_unavailable)(inner) + outer = RetryErrors(_Rendezvous, _retry_on_unavailable)(inner) return outer() def test_log_text(self):