Updating Bigtable system test to work with emulator.

Currently does not work as-is since most of the methods
(most notable row.commit()) result in UNAVAILABLE from
the backend.

Author: dhermes

gcloud/bigtable/client.py

@@ -27,8 +27,11 @@
 """
 
 
+import os
+
 from pkg_resources import get_distribution
 
+from gcloud._helpers import make_insecure_stub
 from gcloud._helpers import make_secure_stub
 from gcloud.bigtable._generated import bigtable_instance_admin_pb2
 from gcloud.bigtable._generated import bigtable_pb2
@@ -40,6 +43,7 @@
 from gcloud.client import _ClientFactoryMixin
 from gcloud.client import _ClientProjectMixin
 from gcloud.credentials import get_credentials
+from gcloud.environment_vars import BIGTABLE_EMULATOR
 
 
 TABLE_ADMIN_HOST = 'bigtableadmin.googleapis.com'
@@ -74,8 +78,12 @@ def _make_data_stub(client):
     :rtype: :class:`._generated.bigtable_pb2.BigtableStub`
     :returns: A gRPC stub object.
     """
-    return make_secure_stub(client.credentials, client.user_agent,
-                            bigtable_pb2.BigtableStub, DATA_API_HOST)
+    if client.emulator_host is None:
+        return make_secure_stub(client.credentials, client.user_agent,
+                                bigtable_pb2.BigtableStub, DATA_API_HOST)
+    else:
+        return make_insecure_stub(bigtable_pb2.BigtableStub,
+                                  client.emulator_host)
 
 
 def _make_instance_stub(client):
@@ -87,10 +95,15 @@ def _make_instance_stub(client):
     :rtype: :class:`.bigtable_instance_admin_pb2.BigtableInstanceAdminStub`
     :returns: A gRPC stub object.
     """
-    return make_secure_stub(
-        client.credentials, client.user_agent,
-        bigtable_instance_admin_pb2.BigtableInstanceAdminStub,
-        INSTANCE_ADMIN_HOST)
+    if client.emulator_host is None:
+        return make_secure_stub(
+            client.credentials, client.user_agent,
+            bigtable_instance_admin_pb2.BigtableInstanceAdminStub,
+            INSTANCE_ADMIN_HOST)
+    else:
+        return make_insecure_stub(
+            bigtable_instance_admin_pb2.BigtableInstanceAdminStub,
+            client.emulator_host)
 
 
 def _make_operations_stub(client):
@@ -105,9 +118,13 @@ def _make_operations_stub(client):
     :rtype: :class:`._generated.operations_grpc_pb2.OperationsStub`
     :returns: A gRPC stub object.
     """
-    return make_secure_stub(client.credentials, client.user_agent,
-                            operations_grpc_pb2.OperationsStub,
-                            OPERATIONS_API_HOST)
+    if client.emulator_host is None:
+        return make_secure_stub(client.credentials, client.user_agent,
+                                operations_grpc_pb2.OperationsStub,
+                                OPERATIONS_API_HOST)
+    else:
+        return make_insecure_stub(operations_grpc_pb2.OperationsStub,
+                                  client.emulator_host)
 
 
 def _make_table_stub(client):
@@ -119,9 +136,14 @@ def _make_table_stub(client):
     :rtype: :class:`.bigtable_instance_admin_pb2.BigtableTableAdminStub`
     :returns: A gRPC stub object.
     """
-    return make_secure_stub(client.credentials, client.user_agent,
-                            bigtable_table_admin_pb2.BigtableTableAdminStub,
-                            TABLE_ADMIN_HOST)
+    if client.emulator_host is None:
+        return make_secure_stub(client.credentials, client.user_agent,
+                                bigtable_table_admin_pb2.BigtableTableAdminStub,
+                                TABLE_ADMIN_HOST)
+    else:
+        return make_insecure_stub(
+            bigtable_table_admin_pb2.BigtableTableAdminStub,
+            client.emulator_host)
 
 
 class Client(_ClientFactoryMixin, _ClientProjectMixin):
@@ -192,6 +214,7 @@ def __init__(self, project=None, credentials=None,
             pass
         self._credentials = credentials
         self.user_agent = user_agent
+        self.emulator_host = os.getenv(BIGTABLE_EMULATOR)
 
         # Create gRPC stubs for making requests.
         self._data_stub = _make_data_stub(self)

gcloud/environment_vars.py

@@ -33,6 +33,9 @@
 PUBSUB_EMULATOR = 'PUBSUB_EMULATOR_HOST'
 """Environment variable defining host for Pub/Sub emulator."""
 
+BIGTABLE_EMULATOR = 'BIGTABLE_EMULATOR_HOST'
+"""Environment variable defining host for Bigtable emulator."""
+
 CREDENTIALS = 'GOOGLE_APPLICATION_CREDENTIALS'
 """Environment variable defining location of Google credentials."""
 

system_tests/bigtable.py

@@ -14,6 +14,7 @@
 
 import datetime
 import operator
+import os
 
 import unittest
 
@@ -29,10 +30,12 @@
 from gcloud.bigtable.row_filters import RowFilterUnion
 from gcloud.bigtable.row_data import Cell
 from gcloud.bigtable.row_data import PartialRowData
+from gcloud.environment_vars import BIGTABLE_EMULATOR
 from gcloud.environment_vars import TESTS_PROJECT
 
 from retry import RetryErrors
 from retry import RetryResult
+from system_test_utils import EmulatorCreds
 from system_test_utils import unique_resource_id
 
 
@@ -61,6 +64,7 @@ class Config(object):
     """
     CLIENT = None
     INSTANCE = None
+    IN_EMULATOR = False
 
 
 def _wait_until_complete(operation, max_attempts=5):
@@ -93,29 +97,43 @@ def _retry_on_unavailable(exc):
 def setUpModule():
     from grpc._channel import _Rendezvous
     _helpers.PROJECT = TESTS_PROJECT
-    Config.CLIENT = Client(admin=True)
+    Config.IN_EMULATOR = os.getenv(BIGTABLE_EMULATOR) is not None
+
+    if Config.IN_EMULATOR:
+        credentials = EmulatorCreds()
+        Config.CLIENT = Client(credentials=credentials, admin=True)
+    else:
+        Config.CLIENT = Client(admin=True)
+
     Config.INSTANCE = Config.CLIENT.instance(INSTANCE_ID, LOCATION_ID)
-    retry = RetryErrors(_Rendezvous, error_predicate=_retry_on_unavailable)
-    instances, failed_locations = retry(Config.CLIENT.list_instances)()
 
-    if len(failed_locations) != 0:
-        raise ValueError('List instances failed in module set up.')
+    if not Config.IN_EMULATOR:
+        retry = RetryErrors(_Rendezvous, error_predicate=_retry_on_unavailable)
+        instances, failed_locations = retry(Config.CLIENT.list_instances)()
+
+        if len(failed_locations) != 0:
+            raise ValueError('List instances failed in module set up.')
 
-    EXISTING_INSTANCES[:] = instances
+        EXISTING_INSTANCES[:] = instances
 
-    # After listing, create the test instance.
-    created_op = Config.INSTANCE.create()
-    if not _wait_until_complete(created_op):
-        raise RuntimeError('Instance creation exceed 5 seconds.')
+        # After listing, create the test instance.
+        created_op = Config.INSTANCE.create()
+        if not _wait_until_complete(created_op):
+            raise RuntimeError('Instance creation exceed 5 seconds.')
 
 
 def tearDownModule():
-    Config.INSTANCE.delete()
+    if not Config.IN_EMULATOR:
+        Config.INSTANCE.delete()
 
 
 class TestInstanceAdminAPI(unittest.TestCase):
 
     def setUp(self):
+        if Config.IN_EMULATOR:
+            self.skipTest(
+                'Instance Admin API not supported by Bigtable emulator')
+
         self.instances_to_delete = []
 
     def tearDown(self):
@@ -186,9 +204,14 @@ def setUpClass(cls):
 
     @classmethod
     def tearDownClass(cls):
-        cls._table.delete()
+        if not Config.IN_EMULATOR:
+            cls._table.delete()
 
     def setUp(self):
+        if Config.IN_EMULATOR:
+            self.skipTest(
+                'Table Admin API not supported by Bigtable emulator')
+
         self.tables_to_delete = []
 
     def tearDown(self):
@@ -291,8 +314,8 @@ def setUpClass(cls):
 
     @classmethod
     def tearDownClass(cls):
-        # Will also delete any data contained in the table.
-        cls._table.delete()
+        if not Config.IN_EMULATOR:
+            cls._table.delete()
 
     def setUp(self):
         self.rows_to_delete = []

system_tests/run_emulator.py

@@ -25,21 +25,24 @@
 
 import psutil
 
+from gcloud.environment_vars import BIGTABLE_EMULATOR
 from gcloud.environment_vars import GCD_DATASET
 from gcloud.environment_vars import GCD_HOST
 from gcloud.environment_vars import PUBSUB_EMULATOR
 from run_system_test import run_module_tests
 
 
 PACKAGE_INFO = {
+    'bigtable': (BIGTABLE_EMULATOR,),
     'datastore': (GCD_DATASET, GCD_HOST),
-    'pubsub': (PUBSUB_EMULATOR,)
+    'pubsub': (PUBSUB_EMULATOR,),
 }
 EXTRA = {
     'datastore': ('--no-legacy',),
 }
 _DS_READY_LINE = '[datastore] Dev App Server is now running.\n'
 _PS_READY_LINE_PREFIX = '[pubsub] INFO: Server started, listening on '
+_BT_READY_LINE_PREFIX = '[bigtable] Cloud Bigtable emulator running on '
 
 
 def get_parser():
@@ -51,7 +54,7 @@ def get_parser():
     parser = argparse.ArgumentParser(
         description='Run GCloud system tests against local emulator.')
     parser.add_argument('--package', dest='package',
-                        choices=('datastore', 'pubsub'),
+                        choices=sorted(PACKAGE_INFO.keys()),
                         default='datastore', help='Package to be tested.')
     return parser
 
@@ -95,16 +98,33 @@ def datastore_wait_ready(popen):
         emulator_ready = popen.stderr.readline() == _DS_READY_LINE
 
 
-def pubsub_wait_ready(popen):
-    """Wait until the pubsub emulator is ready to use.
+def _prefix_wait_ready(popen, prefix):
+    """Wait until a process outputs a line with a given prefix.
 
     :type popen: :class:`subprocess.Popen`
     :param popen: An open subprocess to interact with.
     """
     emulator_ready = False
     while not emulator_ready:
-        emulator_ready = popen.stderr.readline().startswith(
-            _PS_READY_LINE_PREFIX)
+        emulator_ready = popen.stderr.readline().startswith(prefix)
+
+
+def pubsub_wait_ready(popen):
+    """Wait until the pubsub emulator is ready to use.
+
+    :type popen: :class:`subprocess.Popen`
+    :param popen: An open subprocess to interact with.
+    """
+    _prefix_wait_ready(popen, _PS_READY_LINE_PREFIX)
+
+
+def bigtable_wait_ready(popen):
+    """Wait until the Bigtable emulator is ready to use.
+
+    :type popen: :class:`subprocess.Popen`
+    :param popen: An open subprocess to interact with.
+    """
+    _prefix_wait_ready(popen, _BT_READY_LINE_PREFIX)
 
 
 def wait_ready(package, popen):
@@ -117,14 +137,16 @@ def wait_ready(package, popen):
     :param popen: An open subprocess to interact with.
 
     :raises: :class:`KeyError` if the ``package`` is not among
-             ``datastore``, ``pubsub``.
+             ``datastore``, ``pubsub``, ``bigtable``.
     """
     if package == 'datastore':
         datastore_wait_ready(popen)
     elif package == 'pubsub':
         pubsub_wait_ready(popen)
+    elif package == 'bigtable':
+        bigtable_wait_ready(popen)
     else:
-        raise KeyError('')
+        raise KeyError(package)
 
 
 def cleanup(pid):

tox.ini

@@ -137,3 +137,11 @@ commands =
     python {toxinidir}/system_tests/run_emulator.py --package=pubsub
 passenv = GCLOUD_*
 deps = {[testenv:datastore-emulator]deps}
+
+[testenv:bigtable-emulator]
+basepython =
+    python2.7
+commands =
+    python {toxinidir}/system_tests/run_emulator.py --package=bigtable
+passenv = GCLOUD_*
+deps = {[testenv:datastore-emulator]deps}