IntervalDtype

Author: jreback

doc/source/whatsnew/v0.20.0.txt

@@ -11,6 +11,7 @@ Highlights include:
 
 - Building pandas for development now requires ``cython >= 0.23`` (:issue:`14831`)
 - The ``.ix`` indexer has been deprecated, see :ref:`here <whatsnew_0200.api_breaking.deprecate_ix>`
+- Addition of an ``IntervalIndex`` and ``Interval`` scalar type, see :ref:`here <whatsnew_0200.enhancements.intervalindex>`
 
 Check the :ref:`API Changes <whatsnew_0200.api_breaking>` and :ref:`deprecations <whatsnew_0200.deprecations>` before updating.
 
@@ -96,6 +97,9 @@ support for bz2 compression in the python 2 c-engine improved (:issue:`14874`).
 
 .. _whatsnew_0200.enhancements.uint64_support:
 
+Enanced UInt64 Support
+^^^^^^^^^^^^^^^^^^^^^^
+
 Pandas has significantly improved support for operations involving unsigned,
 or purely non-negative, integers. Previously, handling these integers would
 result in improper rounding or data-type casting, leading to incorrect results.
@@ -114,6 +118,36 @@ Notably, a new numerical index, ``UInt64Index``, has been created (:issue:`14937
 - Bug in ``pd.unique()`` in which unsigned 64-bit integers were causing overflow (:issue:`14915`)
 - Bug in ``pd.value_counts()`` in which unsigned 64-bit integers were being erroneously truncated in the output (:issue:`14934`)
 
+.. _whatsnew_0200.enhancements.intervalindex:
+
+IntervalIndex
+^^^^^^^^^^^^^
+
+pandas has gain an ``IntervalIndex`` with its own dtype, ``interval`` as well as the ``Interval`` scalar type. These allow first-class support for interval
+notation, specifically as return type for ``pd.cut`` and ``pd.qcut``. (:issue:`7640`, :issue:`8625`)
+
+**Previous behavior**:
+
+.. code-block:: ipython
+
+   In [2]: pd.cut(range(3), 2)
+   Out[2]:
+   [(-0.002, 1], (-0.002, 1], (1, 2]]
+   Categories (2, object): [(-0.002, 1] < (1, 2]]
+
+   # the returned categories are strings, representing Intervals
+   In [3]: pd.cut(range(3), 2).categories
+   Out[3]: Index(['(-0.002, 1]', '(1, 2]'], dtype='object')
+
+**New behavior**:
+
+.. ipython:: python
+
+   c = pd.cut(range(3), 2)
+   c
+   c.categories
+   pd.api.types.is_interval_dtype(c.categories)
+
 .. _whatsnew_0200.enhancements.other:
 
 Other enhancements

pandas/api/tests/test_api.py

@@ -156,6 +156,7 @@ class TestTypes(Base, tm.TestCase):
                'is_string_dtype', 'is_signed_integer_dtype',
                'is_timedelta64_dtype', 'is_timedelta64_ns_dtype',
                'is_unsigned_integer_dtype', 'is_period',
+               'is_interval', 'is_interval_dtype',
                'is_period_dtype', 'is_re', 'is_re_compilable',
                'is_dict_like', 'is_iterator',
                'is_list_like', 'is_hashable',

pandas/core/algorithms.py

@@ -8,13 +8,14 @@
 
 from pandas import compat, lib, tslib, _np_version_under1p8
 from pandas.types.cast import _maybe_promote
-from pandas.types.generic import ABCSeries, ABCIndex, ABCIntervalIndex
+from pandas.types.generic import ABCSeries, ABCIndex
 from pandas.types.common import (is_unsigned_integer_dtype,
                                  is_signed_integer_dtype,
                                  is_integer_dtype,
                                  is_int64_dtype,
                                  is_categorical_dtype,
                                  is_extension_type,
+                                 is_interval_dtype,
                                  is_datetimetz,
                                  is_period_dtype,
                                  is_period_arraylike,
@@ -1253,10 +1254,7 @@ def take_nd(arr, indexer, axis=0, out=None, fill_value=np.nan, mask_info=None,
                            allow_fill=allow_fill)
     elif is_datetimetz(arr):
         return arr.take(indexer, fill_value=fill_value, allow_fill=allow_fill)
-    elif isinstance(arr, ABCIntervalIndex):
-        # TODO: we need to be sure we are taking on an actual IntervalIndex
-        # this is 'hacky' until we have a first class dtype
-        # ideally will use is_interval_dtype here
+    elif is_interval_dtype(arr):
         return arr.take(indexer, fill_value=fill_value, allow_fill=allow_fill)
 
     if indexer is None:

pandas/indexes/base.py

@@ -3516,10 +3516,8 @@ def _evaluate_compare(self, other):
                 if needs_i8_conversion(self) and needs_i8_conversion(other):
                     return self._evaluate_compare(other, op)
 
-                # TODO: super hack
                 if (is_object_dtype(self) and
-                        self.nlevels == 1 and not
-                        is_interval_dtype(self)):
+                        self.nlevels == 1):
 
                     # don't pass MultiIndex
                     with np.errstate(all='ignore'):

pandas/indexes/interval.py

@@ -3,6 +3,7 @@
 import numpy as np
 
 from pandas.types.missing import notnull, isnull
+from pandas.types.dtypes import IntervalDtype
 from pandas.types.common import (_ensure_platform_int,
                                  is_datetime_or_timedelta_dtype,
                                  is_integer_dtype,
@@ -93,7 +94,7 @@ class IntervalIndex(IntervalMixin, Index):
     _engine = None  # disable it
 
     def __new__(cls, left, right=None, closed='right', mask=None,
-                name=None, copy=False, fastpath=False):
+                name=None, copy=False, dtype=None, fastpath=False):
 
         if right is None:
 
@@ -215,6 +216,7 @@ def from_breaks(cls, breaks, closed='right', name=None):
                       right=[1, 2, 3],
                       closed='right')
         """
+        breaks = np.asarray(breaks)
         mask = isnull(breaks[:-1])
         return cls(breaks[:-1], breaks[1:], closed, mask=mask, name=name)
 
@@ -357,7 +359,7 @@ def astype(self, dtype, copy=True):
 
     @cache_readonly
     def dtype(self):
-        return np.dtype('O')
+        return IntervalDtype.construct_from_string(str(self.left.dtype))
 
     @property
     def inferred_type(self):
@@ -645,6 +647,7 @@ def _format_attrs(self):
                  ('closed', repr(self.closed))]
         if self.name is not None:
             attrs.append(('name', default_pprint(self.name)))
+        attrs.append(('dtype', "'%s'" % self.dtype))
         return attrs
 
     def _format_space(self):

pandas/tests/indexes/test_interval.py

@@ -155,9 +155,10 @@ def test_equals(self):
         idx = self.index
         self.assertTrue(idx.equals(idx))
         self.assertTrue(idx.equals(idx.copy()))
-        self.assertTrue(idx.equals(idx.astype(object)))
-        self.assertTrue(idx.equals(np.array(idx)))
-        self.assertTrue(idx.equals(list(idx)))
+
+        self.assertFalse(idx.equals(idx.astype(object)))
+        self.assertFalse(idx.equals(np.array(idx)))
+        self.assertFalse(idx.equals(list(idx)))
 
         self.assertFalse(idx.equals([1, 2]))
         self.assertFalse(idx.equals(np.array([1, 2])))
@@ -175,7 +176,8 @@ def test_astype(self):
 
         result = idx.astype(object)
         tm.assert_index_equal(result, Index(idx.values, dtype='object'))
-        self.assertTrue(idx.equals(result))
+        self.assertFalse(idx.equals(result))
+        self.assertTrue(idx.equals(IntervalIndex.from_intervals(result)))
 
         result = idx.astype('interval')
         tm.assert_index_equal(result, idx)
@@ -229,8 +231,10 @@ def test_monotonic_and_unique(self):
         self.assertTrue(idx.is_monotonic)
 
     def test_repr(self):
-        expected = ("IntervalIndex(left=[0, 1],\n              right=[1, 2],"
-                    "\n              closed='right')")
+        expected = ("IntervalIndex(left=[0, 1],"
+                    "\n              right=[1, 2],"
+                    "\n              closed='right',"
+                    "\n              dtype='interval[int64]')")
         IntervalIndex((0, 1), (1, 2), closed='right')
         self.assertEqual(repr(self.index), expected)
 

pandas/tests/types/test_dtypes.py

@@ -6,7 +6,8 @@
 import pandas as pd
 from pandas import Series, Categorical, IntervalIndex, date_range
 
-from pandas.types.dtypes import DatetimeTZDtype, PeriodDtype, CategoricalDtype
+from pandas.types.dtypes import (DatetimeTZDtype, PeriodDtype,
+                                 IntervalDtype, CategoricalDtype)
 from pandas.types.common import (is_categorical_dtype, is_categorical,
                                  is_datetime64tz_dtype, is_datetimetz,
                                  is_period_dtype, is_period,
@@ -16,7 +17,7 @@
                                  _coerce_to_dtype)
 import pandas.util.testing as tm
 
-_multiprocess_can_split_ = True
+_multiprocess_can_split_ = False
 
 
 class Base(object):
@@ -355,16 +356,104 @@ def test_not_string(self):
         self.assertFalse(is_string_dtype(PeriodDtype('D')))
 
 
-# TODO: inherit from Base
-class TestIntervalDtype(tm.TestCase):
+class TestIntervalDtype(Base, tm.TestCase):
 
     # TODO: placeholder
     def setUp(self):
-        pass
+        self.dtype = IntervalDtype('int64')
+
+    def test_construction(self):
+        with tm.assertRaises(ValueError):
+            IntervalDtype('xx')
+
+        for s in ['interval[int64]', 'Interval[int64]', 'int64']:
+            i = IntervalDtype(s)
+            self.assertEqual(i.subtype, np.dtype('int64'))
+            self.assertTrue(is_interval_dtype(i))
+
+    def test_construction_generic(self):
+        # generic
+        i = IntervalDtype('interval')
+        self.assertIs(i.subtype, None)
+        self.assertTrue(is_interval_dtype(i))
+        self.assertTrue(str(i) == 'interval')
+
+        i = IntervalDtype()
+        self.assertIs(i.subtype, None)
+        self.assertTrue(is_interval_dtype(i))
+        self.assertTrue(str(i) == 'interval')
+
+    def test_subclass(self):
+        a = IntervalDtype('interval[int64]')
+        b = IntervalDtype('interval[int64]')
+
+        self.assertTrue(issubclass(type(a), type(a)))
+        self.assertTrue(issubclass(type(a), type(b)))
 
     def test_is_dtype(self):
+        self.assertTrue(IntervalDtype.is_dtype(self.dtype))
+        self.assertTrue(IntervalDtype.is_dtype('interval'))
+        self.assertTrue(IntervalDtype.is_dtype(IntervalDtype('float64')))
+        self.assertTrue(IntervalDtype.is_dtype(IntervalDtype('int64')))
+        self.assertTrue(IntervalDtype.is_dtype(IntervalDtype(np.int64)))
+
+        self.assertFalse(IntervalDtype.is_dtype('D'))
+        self.assertFalse(IntervalDtype.is_dtype('3D'))
+        self.assertFalse(IntervalDtype.is_dtype('U'))
+        self.assertFalse(IntervalDtype.is_dtype('S'))
+        self.assertFalse(IntervalDtype.is_dtype('foo'))
+        self.assertFalse(IntervalDtype.is_dtype(np.object_))
+        self.assertFalse(IntervalDtype.is_dtype(np.int64))
+        self.assertFalse(IntervalDtype.is_dtype(np.float64))
+
+    def test_identity(self):
+        self.assertEqual(IntervalDtype('interval[int64]'),
+                         IntervalDtype('interval[int64]'))
+
+    def test_coerce_to_dtype(self):
+        self.assertEqual(_coerce_to_dtype('interval[int64]'),
+                         IntervalDtype('interval[int64]'))
+
+    def test_construction_from_string(self):
+        result = IntervalDtype('interval[int64]')
+        self.assertTrue(is_dtype_equal(self.dtype, result))
+        result = IntervalDtype.construct_from_string('interval[int64]')
+        self.assertTrue(is_dtype_equal(self.dtype, result))
+        with tm.assertRaises(TypeError):
+            IntervalDtype.construct_from_string('foo')
+        with tm.assertRaises(TypeError):
+            IntervalDtype.construct_from_string('interval[foo]')
+        with tm.assertRaises(TypeError):
+            IntervalDtype.construct_from_string('foo[int64]')
 
-        self.assertTrue(is_interval_dtype('interval'))
+    def test_equality(self):
+        self.assertTrue(is_dtype_equal(self.dtype, 'interval[int64]'))
+        self.assertTrue(is_dtype_equal(self.dtype, IntervalDtype('int64')))
+        self.assertTrue(is_dtype_equal(self.dtype, IntervalDtype('int64')))
+        self.assertTrue(is_dtype_equal(IntervalDtype('int64'),
+                                       IntervalDtype('int64')))
+
+        self.assertFalse(is_dtype_equal(self.dtype, 'int64'))
+        self.assertFalse(is_dtype_equal(IntervalDtype('int64'),
+                                        IntervalDtype('float64')))
+
+    def test_basic(self):
+        self.assertTrue(is_interval_dtype(self.dtype))
+
+        ii = IntervalIndex.from_breaks(range(3))
+
+        self.assertTrue(is_interval_dtype(ii.dtype))
+        self.assertTrue(is_interval_dtype(ii))
+
+        s = Series(ii, name='A')
+
+        # dtypes
+        # series results in object dtype currently,
+        self.assertFalse(is_interval_dtype(s.dtype))
+        self.assertFalse(is_interval_dtype(s))
+
+    def test_basic_dtype(self):
+        self.assertTrue(is_interval_dtype('interval[int64]'))
         self.assertTrue(is_interval_dtype(IntervalIndex.from_tuples([(0, 1)])))
         self.assertTrue(is_interval_dtype
                         (IntervalIndex.from_breaks(np.arange(4))))
@@ -377,6 +466,7 @@ def test_is_dtype(self):
         self.assertFalse(is_interval_dtype(np.int64))
         self.assertFalse(is_interval_dtype(np.float64))
 
+
 if __name__ == '__main__':
     nose.runmodule(argv=[__file__, '-vvs', '-x', '--pdb', '--pdb-failure'],
                    exit=False)

pandas/tools/tests/test_tile.py

@@ -7,7 +7,7 @@
 from pandas import (Series, Index, isnull,
                     to_datetime, DatetimeIndex, Timestamp,
                     Interval, IntervalIndex, Categorical,
-                    cut, qcut)
+                    cut, qcut, date_range)
 import pandas.util.testing as tm
 
 from pandas.core.algorithms import quantile
@@ -379,6 +379,20 @@ def test_datetime_bin(self):
         result = cut(data, bins=bin_pydatetime)
         tm.assert_series_equal(Series(result), expected)
 
+    def test_datetime_nan(self):
+
+        def f():
+            cut(date_range('20130101', periods=3), bins=[0, 2, 4])
+        self.assertRaises(ValueError, f)
+
+        result = cut(date_range('20130102', periods=5),
+                     bins=date_range('20130101', periods=2))
+        mask = result.categories.isnull()
+        self.assert_numpy_array_equal(mask, np.array([False]))
+        mask = result.isnull()
+        self.assert_numpy_array_equal(
+            mask, np.array([False, True, True, True, True]))
+
 
 def curpath():
     pth, _ = os.path.split(os.path.abspath(__file__))

pandas/tools/tile.py

@@ -120,7 +120,7 @@ def cut(x, bins, right=True, labels=None, retbins=False, precision=3,
 
     else:
         bins = np.asarray(bins)
-        bins = _convert_bin_to_numeric_type(bins)
+        bins = _convert_bin_to_numeric_type(bins, dtype)
         if (np.diff(bins) < 0).any():
             raise ValueError('bins must increase monotonically.')
 
@@ -277,17 +277,33 @@ def _coerce_to_type(x):
     return x, dtype
 
 
-def _convert_bin_to_numeric_type(x):
+def _convert_bin_to_numeric_type(bins, dtype):
     """
     if the passed bin is of datetime/timedelta type,
     this method converts it to integer
+
+    Parameters
+    ----------
+    bins : list-liek of bins
+    dtype : dtype of data
+
+    Raises
+    ------
+    ValueError if bins are not of a compat dtype to dtype
     """
-    dtype = infer_dtype(x)
-    if dtype == 'timedelta' or dtype == 'timedelta64':
-        x = to_timedelta(x).view(np.int64)
-    elif dtype == 'datetime' or dtype == 'datetime64':
-        x = to_datetime(x).view(np.int64)
-    return x
+    bins_dtype = infer_dtype(bins)
+    if is_timedelta64_dtype(dtype):
+        if bins_dtype in ['timedelta', 'timedelta64']:
+            bins = to_timedelta(bins).view(np.int64)
+        else:
+            raise ValueError("bins must be of timedelta64 dtype")
+    elif is_datetime64_dtype(dtype):
+        if bins_dtype in ['datetime', 'datetime64']:
+            bins = to_datetime(bins).view(np.int64)
+        else:
+            raise ValueError("bins must be of datetime64 dtype")
+
+    return bins
 
 
 def _format_labels(bins, precision, right=True,