diff --git a/doc/source/whatsnew/v1.3.0.rst b/doc/source/whatsnew/v1.3.0.rst index 16f76651a65aa..bafaa1fcc6c4d 100644 --- a/doc/source/whatsnew/v1.3.0.rst +++ b/doc/source/whatsnew/v1.3.0.rst @@ -300,6 +300,38 @@ cast to ``dtype=object`` (:issue:`38709`) ser2 +.. _whatsnew_130.notable_bug_fixes.rolling_groupby_column: + +GroupBy.rolling no longer returns grouped-by column in values +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The group-by column will now be dropped from the result of a +``groupby.rolling`` operation (:issue:`32262`) + +.. ipython:: python + + df = pd.DataFrame({"A": [1, 1, 2, 3], "B": [0, 1, 2, 3]}) + df + +*Previous behavior*: + +.. code-block:: ipython + + In [1]: df.groupby("A").rolling(2).sum() + Out[1]: + A B + A + 1 0 NaN NaN + 1 2.0 1.0 + 2 2 NaN NaN + 3 3 NaN NaN + +*New behavior*: + +.. ipython:: python + + df.groupby("A").rolling(2).sum() + .. _whatsnew_130.notable_bug_fixes.rolling_var_precision: Removed artificial truncation in rolling variance and standard deviation diff --git a/pandas/core/window/rolling.py b/pandas/core/window/rolling.py index 0fa49dccda573..b482934dd25d2 100644 --- a/pandas/core/window/rolling.py +++ b/pandas/core/window/rolling.py @@ -558,6 +558,10 @@ def __init__( if _grouper is None: raise ValueError("Must pass a Grouper object.") self._grouper = _grouper + # GH 32262: It's convention to keep the grouping column in + # groupby., but unexpected to users in + # groupby.rolling. + obj = obj.drop(columns=self._grouper.names, errors="ignore") super().__init__(obj, *args, **kwargs) def _apply( diff --git a/pandas/tests/window/test_groupby.py b/pandas/tests/window/test_groupby.py index c3c5bbe460134..5c2f69a9247e9 100644 --- a/pandas/tests/window/test_groupby.py +++ b/pandas/tests/window/test_groupby.py @@ -83,6 +83,8 @@ def test_rolling(self, f): result = getattr(r, f)() expected = g.apply(lambda x: getattr(x.rolling(4), f)()) + # groupby.apply doesn't drop the grouped-by column + expected = expected.drop("A", axis=1) # GH 39732 expected_index = MultiIndex.from_arrays([self.frame["A"], range(40)]) expected.index = expected_index @@ -95,6 +97,8 @@ def test_rolling_ddof(self, f): result = getattr(r, f)(ddof=1) expected = g.apply(lambda x: getattr(x.rolling(4), f)(ddof=1)) + # groupby.apply doesn't drop the grouped-by column + expected = expected.drop("A", axis=1) # GH 39732 expected_index = MultiIndex.from_arrays([self.frame["A"], range(40)]) expected.index = expected_index @@ -111,6 +115,8 @@ def test_rolling_quantile(self, interpolation): expected = g.apply( lambda x: x.rolling(4).quantile(0.4, interpolation=interpolation) ) + # groupby.apply doesn't drop the grouped-by column + expected = expected.drop("A", axis=1) # GH 39732 expected_index = MultiIndex.from_arrays([self.frame["A"], range(40)]) expected.index = expected_index @@ -147,6 +153,8 @@ def test_rolling_apply(self, raw): # reduction result = r.apply(lambda x: x.sum(), raw=raw) expected = g.apply(lambda x: x.rolling(4).apply(lambda y: y.sum(), raw=raw)) + # groupby.apply doesn't drop the grouped-by column + expected = expected.drop("A", axis=1) # GH 39732 expected_index = MultiIndex.from_arrays([self.frame["A"], range(40)]) expected.index = expected_index @@ -442,6 +450,8 @@ def test_groupby_rolling_empty_frame(self): # GH 36197 expected = DataFrame({"s1": []}) result = expected.groupby("s1").rolling(window=1).sum() + # GH 32262 + expected = expected.drop(columns="s1") # GH-38057 from_tuples gives empty object dtype, we now get float/int levels # expected.index = MultiIndex.from_tuples([], names=["s1", None]) expected.index = MultiIndex.from_product( @@ -451,6 +461,8 @@ def test_groupby_rolling_empty_frame(self): expected = DataFrame({"s1": [], "s2": []}) result = expected.groupby(["s1", "s2"]).rolling(window=1).sum() + # GH 32262 + expected = expected.drop(columns=["s1", "s2"]) expected.index = MultiIndex.from_product( [ Index([], dtype="float64"), @@ -503,6 +515,8 @@ def test_groupby_rolling_no_sort(self): columns=["foo", "bar"], index=MultiIndex.from_tuples([(2, 0), (1, 1)], names=["foo", None]), ) + # GH 32262 + expected = expected.drop(columns="foo") tm.assert_frame_equal(result, expected) def test_groupby_rolling_count_closed_on(self): @@ -553,6 +567,8 @@ def test_groupby_rolling_sem(self, func, kwargs): [("a", 0), ("a", 1), ("b", 2), ("b", 3), ("b", 4)], names=["a", None] ), ) + # GH 32262 + expected = expected.drop(columns="a") tm.assert_frame_equal(result, expected) @pytest.mark.parametrize( @@ -666,6 +682,19 @@ def test_groupby_rolling_object_doesnt_affect_groupby_apply(self): assert not g.mutated assert not g.grouper.mutated + @pytest.mark.parametrize( + "columns", [MultiIndex.from_tuples([("A", ""), ("B", "C")]), ["A", "B"]] + ) + def test_by_column_not_in_values(self, columns): + # GH 32262 + df = DataFrame([[1, 0]] * 20 + [[2, 0]] * 12 + [[3, 0]] * 8, columns=columns) + g = df.groupby("A") + original_obj = g.obj.copy(deep=True) + r = g.rolling(4) + result = r.sum() + assert "A" not in result.columns + tm.assert_frame_equal(g.obj, original_obj) + class TestExpanding: def setup_method(self): @@ -680,6 +709,8 @@ def test_expanding(self, f): result = getattr(r, f)() expected = g.apply(lambda x: getattr(x.expanding(), f)()) + # groupby.apply doesn't drop the grouped-by column + expected = expected.drop("A", axis=1) # GH 39732 expected_index = MultiIndex.from_arrays([self.frame["A"], range(40)]) expected.index = expected_index @@ -692,6 +723,8 @@ def test_expanding_ddof(self, f): result = getattr(r, f)(ddof=0) expected = g.apply(lambda x: getattr(x.expanding(), f)(ddof=0)) + # groupby.apply doesn't drop the grouped-by column + expected = expected.drop("A", axis=1) # GH 39732 expected_index = MultiIndex.from_arrays([self.frame["A"], range(40)]) expected.index = expected_index @@ -708,6 +741,8 @@ def test_expanding_quantile(self, interpolation): expected = g.apply( lambda x: x.expanding().quantile(0.4, interpolation=interpolation) ) + # groupby.apply doesn't drop the grouped-by column + expected = expected.drop("A", axis=1) # GH 39732 expected_index = MultiIndex.from_arrays([self.frame["A"], range(40)]) expected.index = expected_index @@ -748,6 +783,8 @@ def test_expanding_apply(self, raw): # reduction result = r.apply(lambda x: x.sum(), raw=raw) expected = g.apply(lambda x: x.expanding().apply(lambda y: y.sum(), raw=raw)) + # groupby.apply doesn't drop the grouped-by column + expected = expected.drop("A", axis=1) # GH 39732 expected_index = MultiIndex.from_arrays([self.frame["A"], range(40)]) expected.index = expected_index diff --git a/pandas/tests/window/test_rolling.py b/pandas/tests/window/test_rolling.py index 0af0bba5f5f8c..cfd09d0842418 100644 --- a/pandas/tests/window/test_rolling.py +++ b/pandas/tests/window/test_rolling.py @@ -719,6 +719,9 @@ def scaled_sum(*args): df = DataFrame(data={"X": range(5)}, index=[0, 0, 1, 1, 1]) expected = DataFrame(data={"X": [0.0, 0.5, 1.0, 1.5, 2.0]}, index=_index) + # GH 40341 + if "by" in grouping: + expected = expected.drop(columns="X", errors="ignore") result = df.groupby(**grouping).rolling(1).apply(scaled_sum, raw=raw, args=(2,)) tm.assert_frame_equal(result, expected)