BUG: bug in groupby on empty frame with multi groupers (#16090)

* TST: separate out groupby/test_nth

* BUG: bug in groupby on empty frame with multi groupers

xref #14784
closes #16064

Author: jreback

doc/source/whatsnew/v0.20.0.txt

@@ -1627,7 +1627,7 @@ Indexing
 - Bug in the HTML display with with a ``MultiIndex`` and truncation (:issue:`14882`)
 - Bug in the display of ``.info()`` where a qualifier (+) would always be displayed with a ``MultiIndex`` that contains only non-strings (:issue:`15245`)
 - Bug in ``pd.concat()`` where the names of ``MultiIndex`` of resulting ``DataFrame`` are not handled correctly when ``None`` is presented in the names of ``MultiIndex`` of input ``DataFrame`` (:issue:`15787`)
-- Bug in ``DataFrame.sort_index()`` and ``Series.sort_index()`` where ``na_position`` doesn't work with a ``MultiIndex`` (:issue:`14784`)
+- Bug in ``DataFrame.sort_index()`` and ``Series.sort_index()`` where ``na_position`` doesn't work with a ``MultiIndex`` (:issue:`14784`, :issue:`16604`)
 
 I/O
 ^^^

pandas/core/indexes/multi.py

@@ -1645,10 +1645,11 @@ def _get_labels_for_sorting(self):
         """
         from pandas.core.categorical import Categorical
 
-        return [Categorical.from_codes(label,
-                                       np.arange(np.array(label).max() + 1,
-                                                 dtype=label.dtype),
-                                       ordered=True)
+        def cats(label):
+            return np.arange(np.array(label).max() + 1 if len(label) else 0,
+                             dtype=label.dtype)
+
+        return [Categorical.from_codes(label, cats(label), ordered=True)
                 for label in self.labels]
 
     def sortlevel(self, level=0, ascending=True, sort_remaining=True):

pandas/tests/groupby/test_groupby.py

@@ -9,7 +9,7 @@
 from numpy import nan
 
 from pandas import (date_range, bdate_range, Timestamp,
-                    isnull, Index, MultiIndex, DataFrame, Series,
+                    Index, MultiIndex, DataFrame, Series,
                     concat, Panel, DatetimeIndex)
 from pandas.errors import UnsupportedFunctionCall, PerformanceWarning
 from pandas.util.testing import (assert_panel_equal, assert_frame_equal,
@@ -87,229 +87,6 @@ def test_select_bad_cols(self):
             # will have to rethink regex if you change message!
             g[['A', 'C']]
 
-    def test_first_last_nth(self):
-        # tests for first / last / nth
-        grouped = self.df.groupby('A')
-        first = grouped.first()
-        expected = self.df.loc[[1, 0], ['B', 'C', 'D']]
-        expected.index = Index(['bar', 'foo'], name='A')
-        expected = expected.sort_index()
-        assert_frame_equal(first, expected)
-
-        nth = grouped.nth(0)
-        assert_frame_equal(nth, expected)
-
-        last = grouped.last()
-        expected = self.df.loc[[5, 7], ['B', 'C', 'D']]
-        expected.index = Index(['bar', 'foo'], name='A')
-        assert_frame_equal(last, expected)
-
-        nth = grouped.nth(-1)
-        assert_frame_equal(nth, expected)
-
-        nth = grouped.nth(1)
-        expected = self.df.loc[[2, 3], ['B', 'C', 'D']].copy()
-        expected.index = Index(['foo', 'bar'], name='A')
-        expected = expected.sort_index()
-        assert_frame_equal(nth, expected)
-
-        # it works!
-        grouped['B'].first()
-        grouped['B'].last()
-        grouped['B'].nth(0)
-
-        self.df.loc[self.df['A'] == 'foo', 'B'] = np.nan
-        self.assertTrue(isnull(grouped['B'].first()['foo']))
-        self.assertTrue(isnull(grouped['B'].last()['foo']))
-        self.assertTrue(isnull(grouped['B'].nth(0)['foo']))
-
-        # v0.14.0 whatsnew
-        df = DataFrame([[1, np.nan], [1, 4], [5, 6]], columns=['A', 'B'])
-        g = df.groupby('A')
-        result = g.first()
-        expected = df.iloc[[1, 2]].set_index('A')
-        assert_frame_equal(result, expected)
-
-        expected = df.iloc[[1, 2]].set_index('A')
-        result = g.nth(0, dropna='any')
-        assert_frame_equal(result, expected)
-
-    def test_first_last_nth_dtypes(self):
-
-        df = self.df_mixed_floats.copy()
-        df['E'] = True
-        df['F'] = 1
-
-        # tests for first / last / nth
-        grouped = df.groupby('A')
-        first = grouped.first()
-        expected = df.loc[[1, 0], ['B', 'C', 'D', 'E', 'F']]
-        expected.index = Index(['bar', 'foo'], name='A')
-        expected = expected.sort_index()
-        assert_frame_equal(first, expected)
-
-        last = grouped.last()
-        expected = df.loc[[5, 7], ['B', 'C', 'D', 'E', 'F']]
-        expected.index = Index(['bar', 'foo'], name='A')
-        expected = expected.sort_index()
-        assert_frame_equal(last, expected)
-
-        nth = grouped.nth(1)
-        expected = df.loc[[3, 2], ['B', 'C', 'D', 'E', 'F']]
-        expected.index = Index(['bar', 'foo'], name='A')
-        expected = expected.sort_index()
-        assert_frame_equal(nth, expected)
-
-        # GH 2763, first/last shifting dtypes
-        idx = lrange(10)
-        idx.append(9)
-        s = Series(data=lrange(11), index=idx, name='IntCol')
-        self.assertEqual(s.dtype, 'int64')
-        f = s.groupby(level=0).first()
-        self.assertEqual(f.dtype, 'int64')
-
-    def test_nth(self):
-        df = DataFrame([[1, np.nan], [1, 4], [5, 6]], columns=['A', 'B'])
-        g = df.groupby('A')
-
-        assert_frame_equal(g.nth(0), df.iloc[[0, 2]].set_index('A'))
-        assert_frame_equal(g.nth(1), df.iloc[[1]].set_index('A'))
-        assert_frame_equal(g.nth(2), df.loc[[]].set_index('A'))
-        assert_frame_equal(g.nth(-1), df.iloc[[1, 2]].set_index('A'))
-        assert_frame_equal(g.nth(-2), df.iloc[[0]].set_index('A'))
-        assert_frame_equal(g.nth(-3), df.loc[[]].set_index('A'))
-        assert_series_equal(g.B.nth(0), df.set_index('A').B.iloc[[0, 2]])
-        assert_series_equal(g.B.nth(1), df.set_index('A').B.iloc[[1]])
-        assert_frame_equal(g[['B']].nth(0),
-                           df.loc[[0, 2], ['A', 'B']].set_index('A'))
-
-        exp = df.set_index('A')
-        assert_frame_equal(g.nth(0, dropna='any'), exp.iloc[[1, 2]])
-        assert_frame_equal(g.nth(-1, dropna='any'), exp.iloc[[1, 2]])
-
-        exp['B'] = np.nan
-        assert_frame_equal(g.nth(7, dropna='any'), exp.iloc[[1, 2]])
-        assert_frame_equal(g.nth(2, dropna='any'), exp.iloc[[1, 2]])
-
-        # out of bounds, regression from 0.13.1
-        # GH 6621
-        df = DataFrame({'color': {0: 'green',
-                                  1: 'green',
-                                  2: 'red',
-                                  3: 'red',
-                                  4: 'red'},
-                        'food': {0: 'ham',
-                                 1: 'eggs',
-                                 2: 'eggs',
-                                 3: 'ham',
-                                 4: 'pork'},
-                        'two': {0: 1.5456590000000001,
-                                1: -0.070345000000000005,
-                                2: -2.4004539999999999,
-                                3: 0.46206000000000003,
-                                4: 0.52350799999999997},
-                        'one': {0: 0.56573799999999996,
-                                1: -0.9742360000000001,
-                                2: 1.033801,
-                                3: -0.78543499999999999,
-                                4: 0.70422799999999997}}).set_index(['color',
-                                                                     'food'])
-
-        result = df.groupby(level=0, as_index=False).nth(2)
-        expected = df.iloc[[-1]]
-        assert_frame_equal(result, expected)
-
-        result = df.groupby(level=0, as_index=False).nth(3)
-        expected = df.loc[[]]
-        assert_frame_equal(result, expected)
-
-        # GH 7559
-        # from the vbench
-        df = DataFrame(np.random.randint(1, 10, (100, 2)), dtype='int64')
-        s = df[1]
-        g = df[0]
-        expected = s.groupby(g).first()
-        expected2 = s.groupby(g).apply(lambda x: x.iloc[0])
-        assert_series_equal(expected2, expected, check_names=False)
-        self.assertTrue(expected.name, 0)
-        self.assertEqual(expected.name, 1)
-
-        # validate first
-        v = s[g == 1].iloc[0]
-        self.assertEqual(expected.iloc[0], v)
-        self.assertEqual(expected2.iloc[0], v)
-
-        # this is NOT the same as .first (as sorted is default!)
-        # as it keeps the order in the series (and not the group order)
-        # related GH 7287
-        expected = s.groupby(g, sort=False).first()
-        result = s.groupby(g, sort=False).nth(0, dropna='all')
-        assert_series_equal(result, expected)
-
-        # doc example
-        df = DataFrame([[1, np.nan], [1, 4], [5, 6]], columns=['A', 'B'])
-        g = df.groupby('A')
-        result = g.B.nth(0, dropna=True)
-        expected = g.B.first()
-        assert_series_equal(result, expected)
-
-        # test multiple nth values
-        df = DataFrame([[1, np.nan], [1, 3], [1, 4], [5, 6], [5, 7]],
-                       columns=['A', 'B'])
-        g = df.groupby('A')
-
-        assert_frame_equal(g.nth(0), df.iloc[[0, 3]].set_index('A'))
-        assert_frame_equal(g.nth([0]), df.iloc[[0, 3]].set_index('A'))
-        assert_frame_equal(g.nth([0, 1]), df.iloc[[0, 1, 3, 4]].set_index('A'))
-        assert_frame_equal(
-            g.nth([0, -1]), df.iloc[[0, 2, 3, 4]].set_index('A'))
-        assert_frame_equal(
-            g.nth([0, 1, 2]), df.iloc[[0, 1, 2, 3, 4]].set_index('A'))
-        assert_frame_equal(
-            g.nth([0, 1, -1]), df.iloc[[0, 1, 2, 3, 4]].set_index('A'))
-        assert_frame_equal(g.nth([2]), df.iloc[[2]].set_index('A'))
-        assert_frame_equal(g.nth([3, 4]), df.loc[[]].set_index('A'))
-
-        business_dates = pd.date_range(start='4/1/2014', end='6/30/2014',
-                                       freq='B')
-        df = DataFrame(1, index=business_dates, columns=['a', 'b'])
-        # get the first, fourth and last two business days for each month
-        key = (df.index.year, df.index.month)
-        result = df.groupby(key, as_index=False).nth([0, 3, -2, -1])
-        expected_dates = pd.to_datetime(
-            ['2014/4/1', '2014/4/4', '2014/4/29', '2014/4/30', '2014/5/1',
-             '2014/5/6', '2014/5/29', '2014/5/30', '2014/6/2', '2014/6/5',
-             '2014/6/27', '2014/6/30'])
-        expected = DataFrame(1, columns=['a', 'b'], index=expected_dates)
-        assert_frame_equal(result, expected)
-
-    def test_nth_multi_index(self):
-        # PR 9090, related to issue 8979
-        # test nth on MultiIndex, should match .first()
-        grouped = self.three_group.groupby(['A', 'B'])
-        result = grouped.nth(0)
-        expected = grouped.first()
-        assert_frame_equal(result, expected)
-
-    def test_nth_multi_index_as_expected(self):
-        # PR 9090, related to issue 8979
-        # test nth on MultiIndex
-        three_group = DataFrame(
-            {'A': ['foo', 'foo', 'foo', 'foo', 'bar', 'bar', 'bar', 'bar',
-                   'foo', 'foo', 'foo'],
-             'B': ['one', 'one', 'one', 'two', 'one', 'one', 'one', 'two',
-                   'two', 'two', 'one'],
-             'C': ['dull', 'dull', 'shiny', 'dull', 'dull', 'shiny', 'shiny',
-                   'dull', 'shiny', 'shiny', 'shiny']})
-        grouped = three_group.groupby(['A', 'B'])
-        result = grouped.nth(0)
-        expected = DataFrame(
-            {'C': ['dull', 'dull', 'dull', 'dull']},
-            index=MultiIndex.from_arrays([['bar', 'bar', 'foo', 'foo'],
-                                          ['one', 'two', 'one', 'two']],
-                                         names=['A', 'B']))
-        assert_frame_equal(result, expected)
-
     def test_group_selection_cache(self):
         # GH 12839 nth, head, and tail should return same result consistently
         df = DataFrame([[1, 2], [1, 4], [5, 6]], columns=['A', 'B'])