Fix tslib issue with new version of pandas (pandas-dev#353)

* Remove unused imports

* starting with pandas 0.20.0 tslib is part of _libs

* Update code to work with pandas 0.20.+

* Sorting between 0.16 and 0.20 versions requires two different methods

* remove deprecated indexer from script

* Convert datetime slicing to loc indexing

Author: bmoscon

.travis.yml

@@ -25,7 +25,7 @@ install:
   - pip install cython --upgrade
   - pip install pymongo --upgrade
   - pip install numpy --upgrade
-  - pip install pandas==0.19.2
+  - pip install pandas --upgrade
   - pip install decorator --upgrade
   - pip install enum34 --upgrade
   - pip install lz4==0.8.2

arctic/chunkstore/date_chunker.py

@@ -29,7 +29,12 @@ def to_chunks(self, df, chunk_size='D', **kwargs):
         elif 'date' in df.columns:
             dates = pd.DatetimeIndex(df.date)
             if not dates.is_monotonic_increasing:
-                df = df.sort(columns='date')
+                # providing support for pandas 0.16.2 to 0.20.x
+                # neither sort method exists in both
+                try:
+                    df = df.sort_values('date')
+                except AttributeError:
+                    df = df.sort(columns='date')
                 dates = pd.DatetimeIndex(df.date)
         else:
             raise Exception("Data must be datetime indexed or have a column named 'date'")

arctic/multi_index.py

@@ -5,7 +5,7 @@
 import logging
 import types
 
-from pandas.tseries.tools import to_datetime as dt
+from pandas import to_datetime as dt
 
 import numpy as np
 import pandas as pd

arctic/scripts/arctic_copy_data.py

@@ -47,8 +47,8 @@ def _copy_symbol(symbols):
                         # No timezone on the original, should we even allow this?
                         preserve_start = preserve_start.replace(tzinfo=None)
                         preserve_end = preserve_end.replace(tzinfo=None)
-                    before = original_data.ix[:preserve_start]
-                    after = original_data.ix[preserve_end:]
+                    before = original_data.loc[:preserve_start]
+                    after = original_data[preserve_end:]
                     new_data = before.append(new_data).append(after)
 
                 mt.write(symbol, new_data, metadata=version.metadata)

arctic/serialization/numpy_records.py

@@ -2,7 +2,10 @@
 import numpy as np
 
 from pandas import DataFrame, MultiIndex, Series, DatetimeIndex
-from pandas.tslib import Timestamp, get_timezone
+try:
+    from pandas._libs.tslib import Timestamp, get_timezone
+except ImportError:
+    from pandas.tslib import Timestamp, get_timezone
 
 
 log = logging.getLogger(__name__)

arctic/store/_pandas_ndarray_store.py

@@ -3,8 +3,6 @@
 
 from bson.binary import Binary
 from pandas import DataFrame, Series, Panel
-from pandas.tslib import Timestamp, get_timezone
-
 import numpy as np
 
 from arctic.serialization.numpy_records import SeriesSerializer, DataFrameSerializer

arctic/tickstore/tickstore.py

@@ -343,7 +343,7 @@ def read(self, symbol, date_range=None, columns=None, include_images=False, allo
             rtn = rtn.sort_index(kind='mergesort')
         if date_range:
             # FIXME: support DateRange.interval...
-            rtn = rtn.ix[date_range.start:date_range.end]
+            rtn = rtn.loc[date_range.start:date_range.end]
         return rtn
 
     def _pad_and_fix_dtypes(self, cols, column_dtypes):

setup.py

@@ -96,7 +96,7 @@ def run_tests(self):
                       "enum34",
                       "lz4<=0.8.2",
                       "mockextras",
-                      "pandas<=0.19.2",
+                      "pandas",
                       "pymongo>=3.0",
                       "python-dateutil",
                       "pytz",

tests/integration/chunkstore/test_chunkstore.py

@@ -1354,7 +1354,11 @@ def test_unsorted_date_col(chunkstore_lib):
                         'vals': range(2)})
 
     chunkstore_lib.write('test_symbol', df)
-    assert_frame_equal(df.sort(columns='date').reset_index(drop=True), chunkstore_lib.read('test_symbol'))
+    try:
+        df = df.sort_values('date')
+    except AttributeError:
+        df = df.sort(columns='date')
+    assert_frame_equal(df.reset_index(drop=True), chunkstore_lib.read('test_symbol'))
     chunkstore_lib.update('test_symbol', df2)
     assert_frame_equal(chunkstore_lib.read('test_symbol'), 
                        pd.DataFrame({'date': pd.date_range('2016-8-31',

tests/integration/store/test_pandas_store.py

@@ -525,7 +525,7 @@ def panel(i1, i2, i3):
                  list(rrule(DAILY, count=i3, dtstart=dt(1970, 1, 1), interval=1)))
 
 
-@pytest.mark.skipif(pd.__version__ >= '0.18.0', reason="issue #115")
+@pytest.mark.xfail(pd.__version__ >= '0.18.0', reason="see issue #115")
 @pytest.mark.parametrize("df_size", list(itertools.combinations_with_replacement([1, 2, 4], r=3)))
 def test_panel_save_read(library, df_size):
     '''Note - empties are not tested here as they don't work!'''
@@ -540,6 +540,7 @@ def test_panel_save_read(library, df_size):
                 str(pn.axes[i].names) + "!=" + str(pn.axes[i].names)
 
 
+@pytest.mark.xfail(pd.__version__ >= '0.20.0', reason='Panel is deprecated')
 def test_panel_save_read_with_nans(library):
     '''Ensure that nan rows are not dropped when calling to_frame.'''
     df1 = DataFrame(data=np.arange(4).reshape((2, 2)), index=['r1', 'r2'], columns=['c1', 'c2'])
@@ -747,15 +748,15 @@ def test_daterange_append(library):
     saved_arr = library.read('MYARR').data
     assert_frame_equal(df, saved_arr, check_names=False)
     # append two more rows
-    rows = df.ix[-2:].copy()
+    rows = df.iloc[-2:].copy()
     rows.index = rows.index + dtd(days=1)
     library.append('MYARR', rows)
     # assert we can rows back out
     assert_frame_equal(rows, library.read('MYARR', date_range=DateRange(rows.index[0])).data)
     # assert we can read back the first array
     assert_frame_equal(df, library.read('MYARR', date_range=DateRange(df.index[0], df.index[-1])).data)
     # append two more rows
-    rows1 = df.ix[-2:].copy()
+    rows1 = df.iloc[-2:].copy()
     rows1.index = rows1.index + dtd(days=2)
     library.append('MYARR', rows1)
     # assert we can read a mix of data

tests/integration/tickstore/test_ts_read.py

@@ -4,7 +4,7 @@
 from numpy.testing.utils import assert_array_equal
 from pandas.util.testing import assert_frame_equal
 import pandas as pd
-from pandas.tseries.index import DatetimeIndex
+from pandas import DatetimeIndex
 from pymongo import ReadPreference
 import pytest
 import pytz

tests/unit/test_multi_index.py

@@ -1,7 +1,7 @@
 from datetime import timedelta
 import itertools
 
-from pandas.tseries.tools import to_datetime as dt
+from pandas import to_datetime as dt
 from pandas.util.testing import assert_frame_equal
 
 from arctic.multi_index import groupby_asof, fancy_group_by, insert_at

tests/unit/tickstore/test_toplevel.py

@@ -118,7 +118,7 @@ def test_slice_pandas_dataframe(start, end, expected_start_index, expected_end_i
     top_level_tick_store = TopLevelTickStore(Mock())
     dates = pd.date_range('20100101', periods=5, freq='2D')
     data = pd.DataFrame(np.random.randn(5, 4), index=dates, columns=list('ABCD'))
-    expected = data.ix[expected_start_index:expected_end_index]
+    expected = data.iloc[expected_start_index:expected_end_index]
     result = top_level_tick_store._slice(data, start, end)
     assert_frame_equal(expected, result), '{}\n{}'.format(expected, result)