From 8bc3881a09e97a3b4f47160e0851da07f7562d1d Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Thu, 19 Feb 2015 11:49:39 -0800 Subject: [PATCH 1/3] Adding lazy loading support for dataset ID. Still need to support connection (and eventually do the same in storage). Again verified only the lazy loading tests used implicit behavior via ```diff diff --git a/gcloud/datastore/_implicit_environ.py b/gcloud/datastore/_implicit_environ.py index 6e61636..6d67be6 100644 --- a/gcloud/datastore/_implicit_environ.py +++ b/gcloud/datastore/_implicit_environ.py @@ -177,6 +177,10 @@ class _LazyProperty(object): self._method = method def __get__(self, obj, objtype): + class FooError(Exception): + pass + raise FooError + if obj is None or objtype is not _DefaultsContainer: return self ``` --- gcloud/datastore/_implicit_environ.py | 78 ++++++++++++++++------ gcloud/datastore/_testing.py | 4 +- gcloud/datastore/test__implicit_environ.py | 53 +++++++++++++++ 3 files changed, 112 insertions(+), 23 deletions(-) diff --git a/gcloud/datastore/_implicit_environ.py b/gcloud/datastore/_implicit_environ.py index ed32b5741b46..2a844b876fe5 100644 --- a/gcloud/datastore/_implicit_environ.py +++ b/gcloud/datastore/_implicit_environ.py @@ -33,22 +33,6 @@ _GCD_DATASET_ENV_VAR_NAME = 'DATASTORE_DATASET' -class _DefaultsContainer(object): - """Container for defaults. - - :type connection: :class:`gcloud.datastore.connection.Connection` - :param connection: Persistent implied connection from environment. - - :type dataset_id: string - :param dataset_id: Persistent implied dataset ID from environment. - """ - - def __init__(self, connection=None, dataset_id=None, implicit=False): - self.implicit = implicit - self.connection = connection - self.dataset_id = dataset_id - - def app_engine_id(): """Gets the App Engine application ID if it can be inferred. @@ -127,6 +111,15 @@ def _determine_default_dataset_id(dataset_id=None): return dataset_id +def _lazy_dataset_id(): + """Alias wrapper for _determine_default_dataset_id. + + Unit test need to be able to replace _determine_default_dataset_id() + so we can't wrap the actual ``function`` object in a ``_LazyProperty``. + """ + return _determine_default_dataset_id() + + def set_default_dataset_id(dataset_id=None): """Set default dataset ID either explicitly or implicitly as fall-back. @@ -150,6 +143,15 @@ def set_default_dataset_id(dataset_id=None): raise EnvironmentError('No dataset ID could be inferred.') +def get_default_dataset_id(): + """Get default dataset ID. + + :rtype: string or ``NoneType`` + :returns: The default dataset ID if one has been set. + """ + return _DEFAULTS.dataset_id + + def get_default_connection(): """Get default connection. @@ -159,13 +161,47 @@ def get_default_connection(): return _DEFAULTS.connection -def get_default_dataset_id(): - """Get default dataset ID. +class _LazyProperty(object): + """Descriptor for lazy loaded property. - :rtype: string or ``NoneType`` - :returns: The default dataset ID if one has been set. + This follows the reify pattern: lazy evaluation and then replacement + after evaluation. + + :type name: string + :param name: The name of the attribute / property being evaluated. + + :type method: callable that takes no arguments + :param method: The method used to evaluate the property. """ - return _DEFAULTS.dataset_id + + def __init__(self, name, method): + self._name = name + self._method = method + + def __get__(self, obj, objtype): + if obj is None or objtype is not _DefaultsContainer: + return self + + setattr(obj, self._name, self._method()) + return getattr(obj, self._name) + + +class _DefaultsContainer(object): + """Container for defaults. + + :type connection: :class:`gcloud.datastore.connection.Connection` + :param connection: Persistent implied connection from environment. + + :type dataset_id: string + :param dataset_id: Persistent implied dataset ID from environment. + """ + + dataset_id = _LazyProperty('dataset_id', _lazy_dataset_id) + + def __init__(self, connection=None, dataset_id=None, implicit=False): + self.connection = connection + if dataset_id is not None or not implicit: + self.dataset_id = dataset_id _DEFAULTS = _DefaultsContainer(implicit=True) diff --git a/gcloud/datastore/_testing.py b/gcloud/datastore/_testing.py index 8a08a818ff29..97e43222c32e 100644 --- a/gcloud/datastore/_testing.py +++ b/gcloud/datastore/_testing.py @@ -24,9 +24,9 @@ def _monkey_defaults(*args, **kwargs): return _Monkey(_implicit_environ, _DEFAULTS=mock_defaults) -def _setup_defaults(test_case): +def _setup_defaults(test_case, *args, **kwargs): test_case._replaced_defaults = _implicit_environ._DEFAULTS - _implicit_environ._DEFAULTS = _DefaultsContainer() + _implicit_environ._DEFAULTS = _DefaultsContainer(*args, **kwargs) def _tear_down_defaults(test_case): diff --git a/gcloud/datastore/test__implicit_environ.py b/gcloud/datastore/test__implicit_environ.py index 3b75c5b08852..833360504bd7 100644 --- a/gcloud/datastore/test__implicit_environ.py +++ b/gcloud/datastore/test__implicit_environ.py @@ -321,6 +321,59 @@ def test_set_implicit_three_env_appengine_and_compute(self): self.assertEqual(connection.timeout, None) +class Test_lazy_loaded_dataset_id(unittest2.TestCase): + + def setUp(self): + from gcloud.datastore._testing import _setup_defaults + _setup_defaults(self, implicit=True) + + def tearDown(self): + from gcloud.datastore._testing import _tear_down_defaults + _tear_down_defaults(self) + + def test_prop_default(self): + from gcloud.datastore import _implicit_environ + from gcloud.datastore._implicit_environ import _DefaultsContainer + from gcloud.datastore._implicit_environ import _LazyProperty + + self.assertTrue(isinstance(_DefaultsContainer.dataset_id, + _LazyProperty)) + self.assertEqual(_implicit_environ._DEFAULTS.dataset_id, None) + + def test_prop_on_wrong_class(self): + from gcloud.datastore._implicit_environ import _LazyProperty + + # Don't actually need a callable for ``method`` since + # __get__ will just return ``self`` in this test. + data_prop = _LazyProperty('dataset_id', None) + + class FakeEnv(object): + dataset_id = data_prop + + self.assertTrue(FakeEnv.dataset_id is data_prop) + self.assertTrue(FakeEnv().dataset_id is data_prop) + + def test_prop_descriptor(self): + from gcloud._testing import _Monkey + from gcloud.datastore import _implicit_environ + + self.assertFalse( + 'dataset_id' in _implicit_environ._DEFAULTS.__dict__) + + DEFAULT = object() + + def mock_default(): + return DEFAULT + + with _Monkey(_implicit_environ, + _determine_default_dataset_id=mock_default): + lazy_loaded = _implicit_environ._DEFAULTS.dataset_id + + self.assertEqual(lazy_loaded, DEFAULT) + self.assertTrue( + 'dataset_id' in _implicit_environ._DEFAULTS.__dict__) + + class _AppIdentity(object): def __init__(self, app_id): From d5d311825a6efb7f7ef2ef0cefdb70d7f0aab138 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Thu, 19 Feb 2015 11:54:31 -0800 Subject: [PATCH 2/3] Fixing broken auth in datastore regression tests. The _DATASET_ENV_VAR_NAME moved from datastore.__init__ to datastore._implicit_environ. --- regression/clear_datastore.py | 3 ++- regression/datastore.py | 3 ++- regression/populate_datastore.py | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/regression/clear_datastore.py b/regression/clear_datastore.py index 88e76a7f6c24..ef67e4181d8d 100644 --- a/regression/clear_datastore.py +++ b/regression/clear_datastore.py @@ -17,9 +17,10 @@ from six.moves import input from gcloud import datastore +from gcloud.datastore import _implicit_environ -datastore._DATASET_ENV_VAR_NAME = 'GCLOUD_TESTS_DATASET_ID' +_implicit_environ._DATASET_ENV_VAR_NAME = 'GCLOUD_TESTS_DATASET_ID' datastore.set_defaults() diff --git a/regression/datastore.py b/regression/datastore.py index efd1a6cbde86..ff871104162c 100644 --- a/regression/datastore.py +++ b/regression/datastore.py @@ -17,12 +17,13 @@ import unittest2 from gcloud import datastore +from gcloud.datastore import _implicit_environ # This assumes the command is being run via tox hence the # repository root is the current directory. from regression import populate_datastore -datastore._DATASET_ENV_VAR_NAME = 'GCLOUD_TESTS_DATASET_ID' +_implicit_environ._DATASET_ENV_VAR_NAME = 'GCLOUD_TESTS_DATASET_ID' datastore.set_defaults() diff --git a/regression/populate_datastore.py b/regression/populate_datastore.py index d61ab4ade389..32467eaf93e4 100644 --- a/regression/populate_datastore.py +++ b/regression/populate_datastore.py @@ -17,9 +17,10 @@ from six.moves import zip from gcloud import datastore +from gcloud.datastore import _implicit_environ -datastore._DATASET_ENV_VAR_NAME = 'GCLOUD_TESTS_DATASET_ID' +_implicit_environ._DATASET_ENV_VAR_NAME = 'GCLOUD_TESTS_DATASET_ID' datastore.set_defaults() From b9d96ed9cad61b47712fa88873ce91e983a0d8b5 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Thu, 19 Feb 2015 12:29:14 -0800 Subject: [PATCH 3/3] Using lazy property decorator instead of test alias. - Ignoring method-hidden (PyLint) so we can decorate methods in a class. - Making _lazy_property_deco(deferred_callable) to decorate and re-use the function name and use it as a callable - Had to hack around Python2.6 differences in `staticmethod` --- gcloud/datastore/_implicit_environ.py | 43 ++++++++++++++-------- gcloud/datastore/test__implicit_environ.py | 23 ++++++++++++ pylintrc_default | 5 +++ 3 files changed, 56 insertions(+), 15 deletions(-) diff --git a/gcloud/datastore/_implicit_environ.py b/gcloud/datastore/_implicit_environ.py index 2a844b876fe5..8399d6b1775d 100644 --- a/gcloud/datastore/_implicit_environ.py +++ b/gcloud/datastore/_implicit_environ.py @@ -111,15 +111,6 @@ def _determine_default_dataset_id(dataset_id=None): return dataset_id -def _lazy_dataset_id(): - """Alias wrapper for _determine_default_dataset_id. - - Unit test need to be able to replace _determine_default_dataset_id() - so we can't wrap the actual ``function`` object in a ``_LazyProperty``. - """ - return _determine_default_dataset_id() - - def set_default_dataset_id(dataset_id=None): """Set default dataset ID either explicitly or implicitly as fall-back. @@ -170,22 +161,40 @@ class _LazyProperty(object): :type name: string :param name: The name of the attribute / property being evaluated. - :type method: callable that takes no arguments - :param method: The method used to evaluate the property. + :type deferred_callable: callable that takes no arguments + :param deferred_callable: The function / method used to evaluate the + property. """ - def __init__(self, name, method): + def __init__(self, name, deferred_callable): self._name = name - self._method = method + self._deferred_callable = deferred_callable def __get__(self, obj, objtype): if obj is None or objtype is not _DefaultsContainer: return self - setattr(obj, self._name, self._method()) + setattr(obj, self._name, self._deferred_callable()) return getattr(obj, self._name) +def _lazy_property_deco(deferred_callable): + """Decorator a method to create a :class:`_LazyProperty`. + + :type deferred_callable: callable that takes no arguments + :param deferred_callable: The function / method used to evaluate the + property. + + :rtype: :class:`_LazyProperty`. + :returns: A lazy property which defers the deferred_callable. + """ + if isinstance(deferred_callable, staticmethod): + # H/T: http://stackoverflow.com/a/9527450/1068170 + # For Python2.7+ deferred_callable.__func__ would suffice. + deferred_callable = deferred_callable.__get__(True) + return _LazyProperty(deferred_callable.__name__, deferred_callable) + + class _DefaultsContainer(object): """Container for defaults. @@ -196,7 +205,11 @@ class _DefaultsContainer(object): :param dataset_id: Persistent implied dataset ID from environment. """ - dataset_id = _LazyProperty('dataset_id', _lazy_dataset_id) + @_lazy_property_deco + @staticmethod + def dataset_id(): + """Return the implicit default dataset ID.""" + return _determine_default_dataset_id() def __init__(self, connection=None, dataset_id=None, implicit=False): self.connection = connection diff --git a/gcloud/datastore/test__implicit_environ.py b/gcloud/datastore/test__implicit_environ.py index 833360504bd7..b28db11d7a09 100644 --- a/gcloud/datastore/test__implicit_environ.py +++ b/gcloud/datastore/test__implicit_environ.py @@ -321,6 +321,29 @@ def test_set_implicit_three_env_appengine_and_compute(self): self.assertEqual(connection.timeout, None) +class Test__lazy_property_deco(unittest2.TestCase): + + def _callFUT(self, deferred_callable): + from gcloud.datastore._implicit_environ import _lazy_property_deco + return _lazy_property_deco(deferred_callable) + + def test_on_function(self): + def test_func(): + pass # pragma: NO COVER never gets called + + lazy_prop = self._callFUT(test_func) + self.assertTrue(lazy_prop._deferred_callable is test_func) + self.assertEqual(lazy_prop._name, 'test_func') + + def test_on_staticmethod(self): + def test_func(): + pass # pragma: NO COVER never gets called + + lazy_prop = self._callFUT(staticmethod(test_func)) + self.assertTrue(lazy_prop._deferred_callable is test_func) + self.assertEqual(lazy_prop._name, 'test_func') + + class Test_lazy_loaded_dataset_id(unittest2.TestCase): def setUp(self): diff --git a/pylintrc_default b/pylintrc_default index 5686333ac6ca..9acb94e94049 100644 --- a/pylintrc_default +++ b/pylintrc_default @@ -73,6 +73,10 @@ ignore = # identical implementation but different docstrings. # - star-args: standard Python idioms for varargs: # ancestor = Query().filter(*order_props) +# - method-hidden: Decorating a method in a class (e.g. in _DefaultsContainer) +# @_lazy_property_deco +# def dataset_id(): +# ... disable = maybe-no-member, no-member, @@ -80,6 +84,7 @@ disable = redefined-builtin, similarities, star-args, + method-hidden, [REPORTS]