From 284bf615565a4c74fbc917396ddfa3b18900140a Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Thu, 10 Sep 2015 18:05:15 -0700 Subject: [PATCH 1/6] BUG: ensure xray works with pandas 0.17.0 We were using some internal routines in pandas to convert object of datetime objects arrays to datetime64. Predictably, these internal routines have now changed, breaking xray. This is definitely my fault but also bad luck -- I had a guard against the internal function dissappearing, but not against the keyword arguments changing. In any case, this fix ensures forwards compatibility with the next release of pandas, which will be coming out next week. --- doc/whats-new.rst | 3 +++ xray/core/common.py | 11 ++--------- xray/core/variable.py | 8 ++++---- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 3fc27217470..a7f22c87439 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -32,6 +32,9 @@ Enhancements Bug fixes ~~~~~~~~~ +- Forwards compatibility with the next release of changes (v0.17.0). + We were using some internal pandas routines for datetime conversion, which + unfortunately have now changed upstream (:issue:`569`). - Aggregation functions now correctly skip ``NaN`` for data for ``complex128`` dtype (:issue:`554`). - Fixed indexing 0d arrays with unicode dtype (:issue:`568`). diff --git a/xray/core/common.py b/xray/core/common.py index 0644f6b72d1..bbb35a5d465 100644 --- a/xray/core/common.py +++ b/xray/core/common.py @@ -418,13 +418,6 @@ def _maybe_promote(dtype): def _possibly_convert_objects(values): """Convert arrays of datetime.datetime and datetime.timedelta objects into - datetime64 and timedelta64 + datetime64 and timedelta64, according to the pandas convention. """ - try: - converter = functools.partial(pd.core.common._possibly_convert_objects, - convert_numeric=False) - except AttributeError: - # our fault for using a private pandas API that has gone missing - # this should do the same coercion (though it will be slower) - converter = lambda x: np.asarray(pd.Series(x)) - return converter(values.ravel()).reshape(values.shape) + return np.asarray(pd.Series(values.ravel())).reshape(values.shape) diff --git a/xray/core/variable.py b/xray/core/variable.py index c84fa6bbc32..f23d8b91d24 100644 --- a/xray/core/variable.py +++ b/xray/core/variable.py @@ -119,11 +119,11 @@ def _as_compatible_data(data, fastpath=False): data = np.asarray(data) if isinstance(data, np.ndarray): - data = common._possibly_convert_objects(data) - if data.dtype.kind == 'M': - # TODO: automatically cast arrays of datetime objects as well + if data.dtype.kind == 'O': + data = common._possibly_convert_objects(data) + elif data.dtype.kind == 'M': data = np.asarray(data, 'datetime64[ns]') - if data.dtype.kind == 'm': + elif data.dtype.kind == 'm': data = np.asarray(data, 'timedelta64[ns]') return _maybe_wrap_data(data) From ae1c38fc3c11b375a47e66576174856be6b600b2 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Thu, 10 Sep 2015 18:44:24 -0700 Subject: [PATCH 2/6] Add Travis build for pandas master --- .travis.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 67b22de9822..b72d5f89c58 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,7 +32,9 @@ matrix: - libnetcdf-dev env: UPDATE_ENV="conda install cython && pip install https://github.com/Unidata/netcdf4-python/archive/master.zip" - python: 2.7 - env: UPDATE_ENV="pip install toolz https://github.com/ContinuumIO/dask/archive/master.zip" + env: UPDATE_ENV="pip install toolz https://github.com/blaze/dask/archive/master.zip" + - python: 2.7 + env: UPDATE_ENV="pip install https://github.com/pydata/pandas/archive/master.zip" allow_failures: - python: 2.7 env: UPDATE_ENV="pip install pydap" @@ -44,7 +46,9 @@ matrix: - libnetcdf-dev env: UPDATE_ENV="conda install cython && pip install https://github.com/Unidata/netcdf4-python/archive/master.zip" - python: 2.7 - env: UPDATE_ENV="pip install toolz https://github.com/ContinuumIO/dask/archive/master.zip" + env: UPDATE_ENV="pip install toolz https://github.com/blaze/dask/archive/master.zip" + - python: 2.7 + env: UPDATE_ENV="pip install https://github.com/pydata/pandas/archive/master.zip" before_install: - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then From 3f776f770976139a76e7775491d37e9421685acc Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Thu, 10 Sep 2015 18:52:01 -0700 Subject: [PATCH 3/6] Fix missing methods dask test --- xray/test/test_dask.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/xray/test/test_dask.py b/xray/test/test_dask.py index 80ad5e54bfa..1c8c5b0b4d3 100644 --- a/xray/test/test_dask.py +++ b/xray/test/test_dask.py @@ -151,12 +151,14 @@ def test_concat(self): def test_missing_methods(self): v = self.lazy_var - with self.assertRaisesRegexp(NotImplementedError, 'dask'): - v.conj() - with self.assertRaisesRegexp(NotImplementedError, 'dask'): + try: v.argsort() - with self.assertRaisesRegexp(NotImplementedError, 'dask'): + except NotImplementedError as err: + self.assertIn('dask', err.message) + try: v[0].item() + except NotImplementedError as err: + self.assertIn('dask', err.message) def test_ufuncs(self): u = self.eager_var From a945d82ed226ab9ba21c150bf4b2648feda99645 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Thu, 10 Sep 2015 18:52:37 -0700 Subject: [PATCH 4/6] Uninstall pandas before install dev version --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index b72d5f89c58..b991897ecb2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,7 +34,7 @@ matrix: - python: 2.7 env: UPDATE_ENV="pip install toolz https://github.com/blaze/dask/archive/master.zip" - python: 2.7 - env: UPDATE_ENV="pip install https://github.com/pydata/pandas/archive/master.zip" + env: UPDATE_ENV="conda remove pandas && pip install https://github.com/pydata/pandas/archive/master.zip" allow_failures: - python: 2.7 env: UPDATE_ENV="pip install pydap" @@ -48,7 +48,7 @@ matrix: - python: 2.7 env: UPDATE_ENV="pip install toolz https://github.com/blaze/dask/archive/master.zip" - python: 2.7 - env: UPDATE_ENV="pip install https://github.com/pydata/pandas/archive/master.zip" + env: UPDATE_ENV="conda remove pandas && pip install https://github.com/pydata/pandas/archive/master.zip" before_install: - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then From c46a2d88a43eeadef31eee67b7d7006498c5f9bd Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Thu, 10 Sep 2015 18:57:23 -0700 Subject: [PATCH 5/6] Fix dask missing methods test again --- xray/test/test_dask.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xray/test/test_dask.py b/xray/test/test_dask.py index 1c8c5b0b4d3..4a3c121b75b 100644 --- a/xray/test/test_dask.py +++ b/xray/test/test_dask.py @@ -154,11 +154,11 @@ def test_missing_methods(self): try: v.argsort() except NotImplementedError as err: - self.assertIn('dask', err.message) + self.assertIn('dask', str(err)) try: v[0].item() except NotImplementedError as err: - self.assertIn('dask', err.message) + self.assertIn('dask', str(err)) def test_ufuncs(self): u = self.eager_var From 12f856363920c23336c608c56a9edfbdce14bd5c Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Thu, 10 Sep 2015 19:09:30 -0700 Subject: [PATCH 6/6] fix dev pandas builds --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index b991897ecb2..a1d490115ec 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,7 +34,7 @@ matrix: - python: 2.7 env: UPDATE_ENV="pip install toolz https://github.com/blaze/dask/archive/master.zip" - python: 2.7 - env: UPDATE_ENV="conda remove pandas && pip install https://github.com/pydata/pandas/archive/master.zip" + env: UPDATE_ENV="conda remove pandas && conda install cython && pip install https://github.com/pydata/pandas/archive/master.zip" allow_failures: - python: 2.7 env: UPDATE_ENV="pip install pydap" @@ -48,7 +48,7 @@ matrix: - python: 2.7 env: UPDATE_ENV="pip install toolz https://github.com/blaze/dask/archive/master.zip" - python: 2.7 - env: UPDATE_ENV="conda remove pandas && pip install https://github.com/pydata/pandas/archive/master.zip" + env: UPDATE_ENV="conda remove pandas && conda install cython && pip install https://github.com/pydata/pandas/archive/master.zip" before_install: - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then