on attempting to do an ordering operation on an unordered Categorical,

show an OrderingWarning but let the operation succeed

Author: jreback

doc/source/categorical.rst

@@ -280,16 +280,12 @@ Sorting and Order
    The default for construction has change in v0.16.0 to ``ordered=False``, from the prior implicit ``ordered=True``
 
 If categorical data is ordered (``s.cat.ordered == True``), then the order of the categories has a
-meaning and certain operations are possible. If the categorical is unordered, a `TypeError` is
-raised.
+meaning and certain operations are possible. If the categorical is unordered, an ``OrderingWarning`` is shown.
 
 .. ipython:: python
 
     s = Series(Categorical(["a","b","c","a"], ordered=False))
-    try:
-        s.sort()
-    except TypeError as e:
-        print("TypeError: " + str(e))
+    s.sort()
     s = Series(["a","b","c","a"]).astype('category',ordered=True)
     s.sort()
     s
@@ -343,8 +339,8 @@ necessarily make the sort order the same as the categories order.
 
 .. note::
 
-    If the `Categorical` is not ordered, ``Series.min()`` and ``Series.max()`` will raise
-    `TypeError`. Numeric operations like ``+``, ``-``, ``*``, ``/`` and operations based on them
+    If the `Categorical` is not ordered, ``Series.min()`` and ``Series.max()`` will show an ``OrderingWarning``
+    Numeric operations like ``+``, ``-``, ``*``, ``/`` and operations based on them
     (e.g.``Series.median()``, which would need to compute the mean between two values if the length
     of an array is even) do not work and raise a `TypeError`.
 

doc/source/whatsnew/v0.16.0.txt

@@ -368,26 +368,7 @@ For ease of creation of series of categorical data, we have added the ability to
    s = Series(["a","b","c","a"]).astype('category',categories=list('abcdef'),ordered=False)
    s
 
-.. warning::
-
-   This simple API change may have suprising effects if a user is relying on the previous defaulted behavior implicity. In particular,
-   sorting operations with a ``Categorical`` will now raise an error:
-
-   .. code-block:: python
-
-      In [1]: df = DataFrame({ 'A' : Series(list('aabc')).astype('category'), 'B' : np.arange(4) })
-
-      In [2]: df['A'].order()
-      TypeError: Categorical not ordered
-      you can use .as_ordered() to change the Categorical to an ordered one
-
-      In [3]: df.groupby('A').sum()
-      ValueError: cannot sort by an unordered Categorical in the grouper
-      you can set sort=False in the groupby expression or
-      make the categorical ordered by using .as_ordered()
-
-   The solution is to make 'A' orderable, e.g. ``df['A'] = df['A'].cat.as_ordered()``
-
+- In prior versions, trying to ``.order()/.argsort()/.searchsorted()`` on an unordered ``Categorical`` would raise a ``TypeError``. This has been relaxed in that the operation will now succeed but show an ``OrderingWarning``. This will perform the ordering in the order of the categories, then in order of appearance for the values within that category. This operation will NOT modify the existing object. (:issue:`9148`)
 
 Indexing Changes
 ~~~~~~~~~~~~~~~~

pandas/core/categorical.py

@@ -24,6 +24,8 @@
 from pandas.core.config import get_option
 from pandas.core import format as fmt
 
+class OrderingWarning(Warning): pass
+
 def _cat_compare_op(op):
     def f(self, other):
         # On python2, you can usually compare any type to any type, and Categoricals can be
@@ -828,6 +830,18 @@ def T(self):
     def nbytes(self):
         return self._codes.nbytes + self._categories.values.nbytes
 
+    def maybe_coerce_as_ordered(self):
+        """
+        if we are not ordered, but try an ordering operation, let it succeed with a warning
+        This may return a new copy of the object
+        """
+        if not self.ordered:
+            warn("Categorical is not ordered\n"
+                 "sort will be in the order of the categories\n"
+                 "you can use .as_ordered() to change the Categorical to an ordered one\n",
+                 OrderingWarning)
+        return self
+
     def searchsorted(self, v, side='left', sorter=None):
         """Find indices where elements should be inserted to maintain order.
 
@@ -847,6 +861,11 @@ def searchsorted(self, v, side='left', sorter=None):
             Optional array of integer indices that sort `self` into ascending
             order. They are typically the result of ``np.argsort``.
 
+        Warns
+        -----
+        OrderingWarning
+            If the `Categorical` is not `ordered`.
+
         Returns
         -------
         indices : array of ints
@@ -878,9 +897,7 @@ def searchsorted(self, v, side='left', sorter=None):
         >>> x.searchsorted(['bread', 'eggs'], side='right', sorter=[0, 1, 2, 3, 5, 4])
         array([3, 5])       # eggs after donuts, after switching milk and donuts
         """
-        if not self.ordered:
-            raise ValueError("Categorical not ordered\n"
-                             "you can use .as_ordered() to change the Categorical to an ordered one\n")
+        self = self.maybe_coerce_as_ordered()
 
         from pandas.core.series import Series
         values_as_codes = self.categories.values.searchsorted(Series(v).values, side)
@@ -1003,13 +1020,17 @@ def argsort(self, ascending=True, **kwargs):
 
         Only ordered Categoricals can be argsorted!
 
+        Warns
+        -----
+        OrderingWarning
+            If the `Categorical` is not `ordered`.
+
         Returns
         -------
         argsorted : numpy array
         """
-        if not self.ordered:
-            raise TypeError("Categorical not ordered\n"
-                            "you can use .as_ordered() to change the Categorical to an ordered one\n")
+
+        self = self.maybe_coerce_as_ordered()
         result = np.argsort(self._codes.copy(), **kwargs)
         if not ascending:
             result = result[::-1]
@@ -1032,6 +1053,11 @@ def order(self, inplace=False, ascending=True, na_position='last'):
             'first' puts NaNs at the beginning
             'last' puts NaNs at the end
 
+        Warns
+        -----
+        OrderingWarning
+            If the `Categorical` is not `ordered`.
+
         Returns
         -------
         y : Category or None
@@ -1040,9 +1066,8 @@ def order(self, inplace=False, ascending=True, na_position='last'):
         --------
         Category.sort
         """
-        if not self.ordered:
-            raise TypeError("Categorical not ordered\n"
-                            "you can use .as_ordered() to change the Categorical to an ordered one\n")
+
+        self = self.maybe_coerce_as_ordered()
         if na_position not in ['last','first']:
             raise ValueError('invalid na_position: {!r}'.format(na_position))
 
@@ -1092,6 +1117,11 @@ def sort(self, inplace=True, ascending=True, na_position='last'):
             'first' puts NaNs at the beginning
             'last' puts NaNs at the end
 
+        Warns
+        -----
+        OrderingWarning
+            If the `Categorical` is not `ordered`.
+
         Returns
         -------
         y : Category or None
@@ -1413,18 +1443,16 @@ def min(self, numeric_only=None, **kwargs):
 
         Only ordered `Categoricals` have a minimum!
 
-        Raises
-        ------
-        TypeError
+        Warns
+        -----
+        OrderingWarning
             If the `Categorical` is not `ordered`.
 
         Returns
         -------
         min : the minimum of this `Categorical`
         """
-        if not self.ordered:
-            raise TypeError("Categorical not ordered\n"
-                            "you can use .as_ordered() to change the Categorical to an ordered one\n")
+        self = self.maybe_coerce_as_ordered()
         if numeric_only:
             good = self._codes != -1
             pointer = self._codes[good].min(**kwargs)
@@ -1441,18 +1469,16 @@ def max(self, numeric_only=None, **kwargs):
 
         Only ordered `Categoricals` have a maximum!
 
-        Raises
-        ------
-        TypeError
+        Warns
+        -----
+        OrderingWarning
             If the `Categorical` is not `ordered`.
 
         Returns
         -------
         max : the maximum of this `Categorical`
         """
-        if not self.ordered:
-            raise TypeError("Categorical not ordered\n"
-                            "you can use .as_ordered() to change the Categorical to an ordered one\n")
+        self = self.maybe_coerce_as_ordered()
         if numeric_only:
             good = self._codes != -1
             pointer = self._codes[good].max(**kwargs)

pandas/core/groupby.py

@@ -1926,10 +1926,7 @@ def __init__(self, index, grouper=None, obj=None, name=None, level=None,
 
                 # must have an ordered categorical
                 if self.sort:
-                    if not self.grouper.ordered:
-                        raise ValueError("cannot sort by an unordered Categorical in the grouper\n"
-                                         "you can set sort=False in the groupby expression or\n"
-                                         "make the categorical ordered by using .set_ordered(True)\n")
+                    self.grouper = self.grouper.maybe_coerce_as_ordered()
 
                 # fix bug #GH8868 sort=False being ignored in categorical groupby
                 else:

pandas/tests/test_categorical.py

@@ -12,7 +12,7 @@
 import pandas as pd
 
 from pandas import Categorical, Index, Series, DataFrame, PeriodIndex, Timestamp
-
+from pandas.core.categorical import OrderingWarning
 import pandas.core.common as com
 import pandas.compat as compat
 import pandas.util.testing as tm
@@ -838,8 +838,10 @@ def test_min_max(self):
 
         # unordered cats have no min/max
         cat = Categorical(["a","b","c","d"], ordered=False)
-        self.assertRaises(TypeError, lambda : cat.min())
-        self.assertRaises(TypeError, lambda : cat.max())
+        with tm.assert_produces_warning(OrderingWarning):
+            cat.min()
+        with tm.assert_produces_warning(OrderingWarning):
+            cat.max()
         cat = Categorical(["a","b","c","d"], ordered=True)
         _min = cat.min()
         _max = cat.max()
@@ -920,7 +922,8 @@ def test_sort(self):
 
         # unordered cats are not sortable
         cat = Categorical(["a","b","b","a"], ordered=False)
-        self.assertRaises(TypeError, lambda : cat.sort())
+        with tm.assert_produces_warning(OrderingWarning):
+            cat.sort()
         cat = Categorical(["a","c","b","d"], ordered=True)
 
         # order
@@ -1594,8 +1597,10 @@ def test_groupby_sort(self):
     def test_min_max(self):
         # unordered cats have no min/max
         cat = Series(Categorical(["a","b","c","d"], ordered=False))
-        self.assertRaises(TypeError, lambda : cat.min())
-        self.assertRaises(TypeError, lambda : cat.max())
+        with tm.assert_produces_warning(OrderingWarning):
+            cat.min()
+        with tm.assert_produces_warning(OrderingWarning):
+            cat.max()
 
         cat = Series(Categorical(["a","b","c","d"], ordered=True))
         _min = cat.min()
@@ -1769,7 +1774,8 @@ def test_sort(self):
 
         # unordered cats are not sortable
         cat = Series(Categorical(["a","b","b","a"], ordered=False))
-        self.assertRaises(TypeError, lambda : cat.sort())
+        with tm.assert_produces_warning(OrderingWarning):
+            cat.sort()
 
         cat = Series(Categorical(["a","c","b","d"], ordered=True))
 
@@ -1803,9 +1809,8 @@ def test_sort(self):
         self.assertEqual(res["sort"].dtype, "category")
         self.assertEqual(res["unsort"].dtype, "category")
 
-        def f():
+        with tm.assert_produces_warning(OrderingWarning):
             df.sort(columns=["unsort"], ascending=False)
-        self.assertRaises(TypeError, f)
 
         # multi-columns sort
         # GH 7848

pandas/tests/test_groupby.py

@@ -10,6 +10,7 @@
 from pandas import date_range,bdate_range, Timestamp
 from pandas.core.index import Index, MultiIndex, Int64Index
 from pandas.core.api import Categorical, DataFrame
+from pandas.core.categorical import OrderingWarning
 from pandas.core.groupby import (SpecificationError, DataError,
                                  _nargsort, _lexsort_indexer)
 from pandas.core.series import Series
@@ -3299,7 +3300,8 @@ def test_groupby_sort_categorical(self):
         result_nosort.index = index
 
         col = 'range'
-        self.assertRaises(ValueError, lambda : df.groupby(col, sort=True).first())
+        with tm.assert_produces_warning(OrderingWarning):
+            df.groupby(col, sort=True).first()
         assert_frame_equal(result_nosort, df.groupby(col, sort=False).first())