From 2201c3c9e8f8aecb841e925f0e68875e06f96c88 Mon Sep 17 00:00:00 2001 From: Richard Shadrach Date: Wed, 31 Mar 2021 20:42:15 -0400 Subject: [PATCH 1/8] WIP --- pandas/core/groupby/categorical.py | 4 ++++ pandas/core/groupby/grouper.py | 1 + pandas/io/json/_json.py | 5 +++++ 3 files changed, 10 insertions(+) diff --git a/pandas/core/groupby/categorical.py b/pandas/core/groupby/categorical.py index c9dd420ec33df..c84787fe3a635 100644 --- a/pandas/core/groupby/categorical.py +++ b/pandas/core/groupby/categorical.py @@ -71,6 +71,10 @@ def recode_for_groupby( # Already sorted according to c.categories; all is fine if sort: + # categories = c.categories.take(c.codes) + # codes = recode_for_categories(c.codes, c.categories, categories) + # dtype = CategoricalDtype(categories, ordered=c.ordered) + # return Categorical(codes, dtype=dtype, fastpath=True), c return c, None # sort=False should order groups in as-encountered order (GH-8868) diff --git a/pandas/core/groupby/grouper.py b/pandas/core/groupby/grouper.py index 51f7b44f6d69d..3fa48fd14c0f2 100644 --- a/pandas/core/groupby/grouper.py +++ b/pandas/core/groupby/grouper.py @@ -516,6 +516,7 @@ def __init__( codes = np.sort(codes) else: codes = np.arange(len(categories)) + # codes = algorithms.unique1d(self.grouper.codes) self._group_index = CategoricalIndex( Categorical.from_codes( diff --git a/pandas/io/json/_json.py b/pandas/io/json/_json.py index b7493ebeadf34..a78f0c859d4ec 100644 --- a/pandas/io/json/_json.py +++ b/pandas/io/json/_json.py @@ -174,6 +174,11 @@ def _format_axes(self): def write(self): iso_dates = self.date_format == "iso" + print(type(self.obj_to_write["data"])) + print(self.obj_to_write["data"]) + print(self.default_handler) + print(dumps(self.obj_to_write["data"])) + print("---") return dumps( self.obj_to_write, orient=self.orient, From 85474cfbe0ee7a56acc4bfbab9bd5ea50955de11 Mon Sep 17 00:00:00 2001 From: Richard Shadrach Date: Thu, 1 Apr 2021 18:43:34 -0400 Subject: [PATCH 2/8] WIP --- pandas/core/dtypes/cast.py | 13 +++++++++---- pandas/core/groupby/groupby.py | 3 ++- pandas/tests/groupby/aggregate/test_aggregate.py | 8 ++++---- pandas/tests/groupby/test_categorical.py | 4 +--- 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index 7a2175a364a8a..4eb9dc5f811ab 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -449,10 +449,15 @@ def maybe_cast_result_dtype(dtype: DtypeObj, how: str) -> DtypeObj: return np.dtype(np.int64) elif isinstance(dtype, (BooleanDtype, _IntegerDtype)): return Int64Dtype() - elif how in ["mean", "median", "var"] and isinstance( - dtype, (BooleanDtype, _IntegerDtype) - ): - return Float64Dtype() + elif how in ["mean", "median", "var"]: + if isinstance( + dtype, (BooleanDtype, _IntegerDtype) + ): + return Float64Dtype() + elif is_float_dtype(dtype): + return dtype + elif is_numeric_dtype(dtype): + return np.dtype(np.float64) return dtype diff --git a/pandas/core/groupby/groupby.py b/pandas/core/groupby/groupby.py index 1ee38834c5758..360dd0010b51f 100644 --- a/pandas/core/groupby/groupby.py +++ b/pandas/core/groupby/groupby.py @@ -1232,7 +1232,8 @@ def _python_agg_general(self, func, *args, **kwargs): assert result is not None key = base.OutputKey(label=name, position=idx) - if is_numeric_dtype(obj.dtype): + print('func:', func) + if not callable(func) and is_numeric_dtype(obj.dtype): result = maybe_downcast_numeric(result, obj.dtype) if self.grouper._filter_empty_groups: diff --git a/pandas/tests/groupby/aggregate/test_aggregate.py b/pandas/tests/groupby/aggregate/test_aggregate.py index fc0b4d86e81bf..a73ba39efd32c 100644 --- a/pandas/tests/groupby/aggregate/test_aggregate.py +++ b/pandas/tests/groupby/aggregate/test_aggregate.py @@ -235,10 +235,10 @@ def test_aggregate_item_by_item(df): # GH5782 # odd comparisons can result here, so cast to make easy - exp = Series(np.array([foo] * K), index=list("BCD"), dtype=np.float64, name="foo") + exp = Series(np.array([foo] * K), index=list("BCD"), name="foo") tm.assert_series_equal(result.xs("foo"), exp) - exp = Series(np.array([bar] * K), index=list("BCD"), dtype=np.float64, name="bar") + exp = Series(np.array([bar] * K), index=list("BCD"), name="bar") tm.assert_almost_equal(result.xs("bar"), exp) def aggfun(ser): @@ -455,7 +455,7 @@ def test_order_aggregate_multiple_funcs(): @pytest.mark.parametrize("dtype", [np.int64, np.uint64]) -@pytest.mark.parametrize("how", ["first", "last", "min", "max", "mean", "median"]) +@pytest.mark.parametrize("how", ["first", "last", "min", "max"]) def test_uint64_type_handling(dtype, how): # GH 26310 df = DataFrame({"x": 6903052872240755750, "y": [1, 2]}) @@ -850,7 +850,7 @@ def test_multiindex_custom_func(func): df = DataFrame(data, columns=MultiIndex.from_arrays([[1, 1, 2], [3, 4, 3]])) result = df.groupby(np.array([0, 1])).agg(func) expected_dict = {(1, 3): {0: 1, 1: 5}, (1, 4): {0: 4, 1: 7}, (2, 3): {0: 2, 1: 1}} - expected = DataFrame(expected_dict) + expected = DataFrame(expected_dict, dtype="float64") tm.assert_frame_equal(result, expected) diff --git a/pandas/tests/groupby/test_categorical.py b/pandas/tests/groupby/test_categorical.py index da438826a939a..dbb02c6856547 100644 --- a/pandas/tests/groupby/test_categorical.py +++ b/pandas/tests/groupby/test_categorical.py @@ -285,8 +285,6 @@ def test_apply(ordered): result = grouped.apply(lambda x: np.mean(x)) tm.assert_frame_equal(result, expected) - # we coerce back to ints - expected = expected.astype("int") result = grouped.mean() tm.assert_frame_equal(result, expected) @@ -371,7 +369,7 @@ def test_observed(observed, using_array_manager): result = groups_double_key.agg("mean") expected = DataFrame( { - "val": [10, 30, 20, 40], + "val": [10., 30, 20, 40], "cat": Categorical( ["a", "a", "b", "b"], categories=["a", "b", "c"], ordered=True ), From b26ed4677bc44729534a71f2b5712d3673a1d704 Mon Sep 17 00:00:00 2001 From: Richard Shadrach Date: Sat, 24 Apr 2021 07:58:41 -0400 Subject: [PATCH 3/8] BUG: groupby sum, mean, var should always be floats --- pandas/core/groupby/categorical.py | 4 ---- pandas/core/groupby/groupby.py | 3 +-- pandas/core/groupby/grouper.py | 1 - pandas/core/groupby/ops.py | 17 ++++++++-------- pandas/io/json/_json.py | 5 ----- pandas/tests/extension/base/groupby.py | 4 ++-- pandas/tests/extension/test_boolean.py | 4 ++-- .../tests/groupby/aggregate/test_aggregate.py | 12 ++++++----- pandas/tests/groupby/aggregate/test_cython.py | 2 ++ pandas/tests/groupby/test_categorical.py | 4 ++-- pandas/tests/groupby/test_function.py | 2 +- pandas/tests/groupby/test_groupby.py | 6 +++--- pandas/tests/resample/test_datetime_index.py | 10 +++++----- pandas/tests/resample/test_period_index.py | 12 +++++------ pandas/tests/reshape/test_pivot.py | 20 ++++++++++++------- 15 files changed, 53 insertions(+), 53 deletions(-) diff --git a/pandas/core/groupby/categorical.py b/pandas/core/groupby/categorical.py index 72d0801abe35b..297681f1e10f5 100644 --- a/pandas/core/groupby/categorical.py +++ b/pandas/core/groupby/categorical.py @@ -71,10 +71,6 @@ def recode_for_groupby( # Already sorted according to c.categories; all is fine if sort: - # categories = c.categories.take(c.codes) - # codes = recode_for_categories(c.codes, c.categories, categories) - # dtype = CategoricalDtype(categories, ordered=c.ordered) - # return Categorical(codes, dtype=dtype, fastpath=True), c return c, None # sort=False should order groups in as-encountered order (GH-8868) diff --git a/pandas/core/groupby/groupby.py b/pandas/core/groupby/groupby.py index 321455e4db658..f579b04db898e 100644 --- a/pandas/core/groupby/groupby.py +++ b/pandas/core/groupby/groupby.py @@ -1241,8 +1241,7 @@ def _python_agg_general(self, func, *args, **kwargs): assert result is not None key = base.OutputKey(label=name, position=idx) - print("func:", func) - if not callable(func) and is_numeric_dtype(obj.dtype): + if is_numeric_dtype(obj.dtype): result = maybe_downcast_numeric(result, obj.dtype) if self.grouper._filter_empty_groups: diff --git a/pandas/core/groupby/grouper.py b/pandas/core/groupby/grouper.py index 834331df712aa..151756b829a1d 100644 --- a/pandas/core/groupby/grouper.py +++ b/pandas/core/groupby/grouper.py @@ -509,7 +509,6 @@ def __init__( codes = np.sort(codes) else: codes = np.arange(len(categories)) - # codes = algorithms.unique1d(self.grouper.codes) self._group_index = CategoricalIndex( Categorical.from_codes( diff --git a/pandas/core/groupby/ops.py b/pandas/core/groupby/ops.py index bfdabaa11520b..b5c37423de69d 100644 --- a/pandas/core/groupby/ops.py +++ b/pandas/core/groupby/ops.py @@ -290,14 +290,15 @@ def get_result_dtype(self, dtype: DtypeObj) -> DtypeObj: return np.dtype(np.int64) elif isinstance(dtype, (BooleanDtype, _IntegerDtype)): return Int64Dtype() - elif how in ["mean", "median", "var"] and isinstance( - dtype, (BooleanDtype, _IntegerDtype) - ): - return Float64Dtype() - elif is_float_dtype(dtype): - return dtype - elif is_numeric_dtype(dtype): - return np.dtype(np.float64) + elif how in ["mean", "median", "var"]: + if isinstance( + dtype, (BooleanDtype, _IntegerDtype) + ): + return Float64Dtype() + elif is_float_dtype(dtype): + return dtype + elif is_numeric_dtype(dtype): + return np.dtype(np.float64) return dtype def uses_mask(self) -> bool: diff --git a/pandas/io/json/_json.py b/pandas/io/json/_json.py index a78f0c859d4ec..b7493ebeadf34 100644 --- a/pandas/io/json/_json.py +++ b/pandas/io/json/_json.py @@ -174,11 +174,6 @@ def _format_axes(self): def write(self): iso_dates = self.date_format == "iso" - print(type(self.obj_to_write["data"])) - print(self.obj_to_write["data"]) - print(self.default_handler) - print(dumps(self.obj_to_write["data"])) - print("---") return dumps( self.obj_to_write, orient=self.orient, diff --git a/pandas/tests/extension/base/groupby.py b/pandas/tests/extension/base/groupby.py index 30b115b9dba6f..dbc519dec1b77 100644 --- a/pandas/tests/extension/base/groupby.py +++ b/pandas/tests/extension/base/groupby.py @@ -25,7 +25,7 @@ def test_groupby_extension_agg(self, as_index, data_for_grouping): _, index = pd.factorize(data_for_grouping, sort=True) index = pd.Index(index, name="B") - expected = pd.Series([3, 1, 4], index=index, name="A") + expected = pd.Series([3.0, 1.0, 4.0], index=index, name="A") if as_index: self.assert_series_equal(result, expected) else: @@ -54,7 +54,7 @@ def test_groupby_extension_no_sort(self, data_for_grouping): _, index = pd.factorize(data_for_grouping, sort=False) index = pd.Index(index, name="B") - expected = pd.Series([1, 3, 4], index=index, name="A") + expected = pd.Series([1.0, 3.0, 4.0], index=index, name="A") self.assert_series_equal(result, expected) def test_groupby_extension_transform(self, data_for_grouping): diff --git a/pandas/tests/extension/test_boolean.py b/pandas/tests/extension/test_boolean.py index 33d82a1d64fb7..8e61b4f4b9bb9 100644 --- a/pandas/tests/extension/test_boolean.py +++ b/pandas/tests/extension/test_boolean.py @@ -272,7 +272,7 @@ def test_groupby_extension_agg(self, as_index, data_for_grouping): _, index = pd.factorize(data_for_grouping, sort=True) index = pd.Index(index, name="B") - expected = pd.Series([3, 1], index=index, name="A") + expected = pd.Series([3.0, 1.0], index=index, name="A") if as_index: self.assert_series_equal(result, expected) else: @@ -301,7 +301,7 @@ def test_groupby_extension_no_sort(self, data_for_grouping): _, index = pd.factorize(data_for_grouping, sort=False) index = pd.Index(index, name="B") - expected = pd.Series([1, 3], index=index, name="A") + expected = pd.Series([1.0, 3.0], index=index, name="A") self.assert_series_equal(result, expected) def test_groupby_extension_transform(self, data_for_grouping): diff --git a/pandas/tests/groupby/aggregate/test_aggregate.py b/pandas/tests/groupby/aggregate/test_aggregate.py index 0053b61006a84..2564aaebf9726 100644 --- a/pandas/tests/groupby/aggregate/test_aggregate.py +++ b/pandas/tests/groupby/aggregate/test_aggregate.py @@ -235,10 +235,10 @@ def test_aggregate_item_by_item(df): # GH5782 # odd comparisons can result here, so cast to make easy - exp = Series(np.array([foo] * K), index=list("BCD"), name="foo") + exp = Series(np.array([foo] * K), index=list("BCD"), dtype=np.float64, name="foo") tm.assert_series_equal(result.xs("foo"), exp) - exp = Series(np.array([bar] * K), index=list("BCD"), name="bar") + exp = Series(np.array([bar] * K), index=list("BCD"), dtype=np.float64, name="bar") tm.assert_almost_equal(result.xs("bar"), exp) def aggfun(ser): @@ -455,14 +455,16 @@ def test_order_aggregate_multiple_funcs(): @pytest.mark.parametrize("dtype", [np.int64, np.uint64]) -@pytest.mark.parametrize("how", ["first", "last", "min", "max"]) +@pytest.mark.parametrize("how", ["first", "last", "min", "max", "mean", "median"]) def test_uint64_type_handling(dtype, how): # GH 26310 df = DataFrame({"x": 6903052872240755750, "y": [1, 2]}) expected = df.groupby("y").agg({"x": how}) df.x = df.x.astype(dtype) result = df.groupby("y").agg({"x": how}) - result.x = result.x.astype(np.int64) + if how not in ("mean", "median"): + # mean and median always result in floats + result.x = result.x.astype(np.int64) tm.assert_frame_equal(result, expected, check_exact=True) @@ -850,7 +852,7 @@ def test_multiindex_custom_func(func): df = DataFrame(data, columns=MultiIndex.from_arrays([[1, 1, 2], [3, 4, 3]])) result = df.groupby(np.array([0, 1])).agg(func) expected_dict = {(1, 3): {0: 1, 1: 5}, (1, 4): {0: 4, 1: 7}, (2, 3): {0: 2, 1: 1}} - expected = DataFrame(expected_dict, dtype="float64") + expected = DataFrame(expected_dict) tm.assert_frame_equal(result, expected) diff --git a/pandas/tests/groupby/aggregate/test_cython.py b/pandas/tests/groupby/aggregate/test_cython.py index 4a8aabe41b754..5aa0bf413a9df 100644 --- a/pandas/tests/groupby/aggregate/test_cython.py +++ b/pandas/tests/groupby/aggregate/test_cython.py @@ -196,6 +196,8 @@ def test_cython_agg_empty_buckets(op, targop, observed): g = df.groupby(pd.cut(df[0], grps), observed=observed) expected = g.agg(lambda x: targop(x)) + if op in ("mean", "median", "var"): + expected = expected.astype(float) tm.assert_frame_equal(result, expected) diff --git a/pandas/tests/groupby/test_categorical.py b/pandas/tests/groupby/test_categorical.py index dbb02c6856547..68031b166282a 100644 --- a/pandas/tests/groupby/test_categorical.py +++ b/pandas/tests/groupby/test_categorical.py @@ -416,7 +416,7 @@ def test_observed_codes_remap(observed): groups_double_key = df.groupby([values, "C2"], observed=observed) idx = MultiIndex.from_arrays([values, [1, 2, 3, 4]], names=["cat", "C2"]) - expected = DataFrame({"C1": [3, 3, 4, 5], "C3": [10, 100, 200, 34]}, index=idx) + expected = DataFrame({"C1": [3.0, 3.0, 4.0, 5.0], "C3": [10.0, 100.0, 200.0, 34.0]}, index=idx) if not observed: expected = cartesian_product_for_groupers( expected, [values.values, [1, 2, 3, 4]], ["cat", "C2"] @@ -1503,7 +1503,7 @@ def test_read_only_category_no_sort(): df = DataFrame( {"a": [1, 3, 5, 7], "b": Categorical([1, 1, 2, 2], categories=Index(cats))} ) - expected = DataFrame(data={"a": [2, 6]}, index=CategoricalIndex([1, 2], name="b")) + expected = DataFrame(data={"a": [2.0, 6.0]}, index=CategoricalIndex([1, 2], name="b")) result = df.groupby("b", sort=False).mean() tm.assert_frame_equal(result, expected) diff --git a/pandas/tests/groupby/test_function.py b/pandas/tests/groupby/test_function.py index f47fc1f4e4a4f..b1f4f76777274 100644 --- a/pandas/tests/groupby/test_function.py +++ b/pandas/tests/groupby/test_function.py @@ -407,7 +407,7 @@ def test_median_empty_bins(observed): bins = pd.cut(df[0], grps) result = df.groupby(bins, observed=observed).median() - expected = df.groupby(bins, observed=observed).agg(lambda x: x.median()) + expected = df.groupby(bins, observed=observed).agg(lambda x: x.median()).astype(float) tm.assert_frame_equal(result, expected) diff --git a/pandas/tests/groupby/test_groupby.py b/pandas/tests/groupby/test_groupby.py index 3f6485be871f1..11f7a1f8678c3 100644 --- a/pandas/tests/groupby/test_groupby.py +++ b/pandas/tests/groupby/test_groupby.py @@ -1237,7 +1237,7 @@ def test_groupby_keys_same_size_as_index(): ) df = DataFrame([["A", 10], ["B", 15]], columns=["metric", "values"], index=index) result = df.groupby([Grouper(level=0, freq=freq), "metric"]).mean() - expected = df.set_index([df.index, "metric"]) + expected = df.set_index([df.index, "metric"]).astype(float) tm.assert_frame_equal(result, expected) @@ -1330,7 +1330,7 @@ def test_groupby_2d_malformed(): d["ones"] = [1, 1] d["label"] = ["l1", "l2"] tmp = d.groupby(["group"]).mean() - res_values = np.array([[0, 1], [0, 1]], dtype=np.int64) + res_values = np.array([[0.0, 1.0], [0.0, 1.0]]) tm.assert_index_equal(tmp.columns, Index(["zeros", "ones"])) tm.assert_numpy_array_equal(tmp.values, res_values) @@ -2023,7 +2023,7 @@ def test_groupby_crash_on_nunique(axis): def test_groupby_list_level(): # GH 9790 - expected = DataFrame(np.arange(0, 9).reshape(3, 3)) + expected = DataFrame(np.arange(0, 9).reshape(3, 3), dtype=float) result = expected.groupby(level=[0]).mean() tm.assert_frame_equal(result, expected) diff --git a/pandas/tests/resample/test_datetime_index.py b/pandas/tests/resample/test_datetime_index.py index bbe9ac6fa8094..89ade3c387391 100644 --- a/pandas/tests/resample/test_datetime_index.py +++ b/pandas/tests/resample/test_datetime_index.py @@ -1064,7 +1064,7 @@ def test_nanosecond_resample_error(): result = r.agg("mean") exp_indx = date_range(start=pd.to_datetime(exp_start), periods=10, freq="100n") - exp = Series(range(len(exp_indx)), index=exp_indx) + exp = Series(range(len(exp_indx)), index=exp_indx, dtype=float) tm.assert_series_equal(result, exp) @@ -1633,15 +1633,15 @@ def test_resample_with_nat(): index_1s = DatetimeIndex( ["1970-01-01 00:00:00", "1970-01-01 00:00:01", "1970-01-01 00:00:02"] ) - frame_1s = DataFrame([3, 7, 11], index=index_1s) + frame_1s = DataFrame([3.0, 7.0, 11.0], index=index_1s) tm.assert_frame_equal(frame.resample("1s").mean(), frame_1s) index_2s = DatetimeIndex(["1970-01-01 00:00:00", "1970-01-01 00:00:02"]) - frame_2s = DataFrame([5, 11], index=index_2s) + frame_2s = DataFrame([5.0, 11.0], index=index_2s) tm.assert_frame_equal(frame.resample("2s").mean(), frame_2s) index_3s = DatetimeIndex(["1970-01-01 00:00:00"]) - frame_3s = DataFrame([7], index=index_3s) + frame_3s = DataFrame([7.0], index=index_3s) tm.assert_frame_equal(frame.resample("3s").mean(), frame_3s) tm.assert_frame_equal(frame.resample("60s").mean(), frame_3s) @@ -1684,7 +1684,7 @@ def f(data, add_arg): # Testing dataframe df = DataFrame({"A": 1, "B": 2}, index=date_range("2017", periods=10)) - result = df.groupby("A").resample("D").agg(f, multiplier) + result = df.groupby("A").resample("D").agg(f, multiplier).astype(float) expected = df.groupby("A").resample("D").mean().multiply(multiplier) tm.assert_frame_equal(result, expected) diff --git a/pandas/tests/resample/test_period_index.py b/pandas/tests/resample/test_period_index.py index e2b13f6a00677..a6491952375a4 100644 --- a/pandas/tests/resample/test_period_index.py +++ b/pandas/tests/resample/test_period_index.py @@ -269,7 +269,7 @@ def test_with_local_timezone_pytz(self): # Index is moved back a day with the timezone conversion from UTC to # Pacific expected_index = period_range(start=start, end=end, freq="D") - offsets.Day() - expected = Series(1, index=expected_index) + expected = Series(1.0, index=expected_index) tm.assert_series_equal(result, expected) def test_resample_with_pytz(self): @@ -279,7 +279,7 @@ def test_resample_with_pytz(self): ) result = s.resample("D").mean() expected = Series( - 2, + 2.0, index=pd.DatetimeIndex( ["2017-01-01", "2017-01-02"], tz="US/Eastern", freq="D" ), @@ -312,7 +312,7 @@ def test_with_local_timezone_dateutil(self): expected_index = ( period_range(start=start, end=end, freq="D", name="idx") - offsets.Day() ) - expected = Series(1, index=expected_index) + expected = Series(1.0, index=expected_index) tm.assert_series_equal(result, expected) def test_resample_nonexistent_time_bin_edge(self): @@ -777,8 +777,8 @@ def test_upsampling_ohlc(self, freq, period_mult, kind): "freq, expected_values", [ ("1s", [3, np.NaN, 7, 11]), - ("2s", [3, int((7 + 11) / 2)]), - ("3s", [int((3 + 7) / 2), 11]), + ("2s", [3, (7 + 11) / 2]), + ("3s", [(3 + 7) / 2, 11]), ], ) def test_resample_with_nat(self, periods, values, freq, expected_values): @@ -798,7 +798,7 @@ def test_resample_with_only_nat(self): pi = PeriodIndex([pd.NaT] * 3, freq="S") frame = DataFrame([2, 3, 5], index=pi, columns=["a"]) expected_index = PeriodIndex(data=[], freq=pi.freq) - expected = DataFrame(index=expected_index, columns=["a"], dtype="int64") + expected = DataFrame(index=expected_index, columns=["a"], dtype="float64") result = frame.resample("1s").mean() tm.assert_frame_equal(result, expected) diff --git a/pandas/tests/reshape/test_pivot.py b/pandas/tests/reshape/test_pivot.py index 3d1c3b81c492f..4bb0283f439ae 100644 --- a/pandas/tests/reshape/test_pivot.py +++ b/pandas/tests/reshape/test_pivot.py @@ -240,13 +240,13 @@ def test_pivot_with_non_observable_dropna(self, dropna): categories=["low", "high"], ordered=True, ), - "B": range(5), + "B": [0.0, 1.0, 2.0, 3.0, 4.0], } ) result = df.pivot_table(index="A", values="B", dropna=dropna) expected = DataFrame( - {"B": [2, 3]}, + {"B": [2.0, 3.0]}, index=Index( Categorical.from_codes( [0, 1], categories=["low", "high"], ordered=True @@ -279,6 +279,8 @@ def test_pivot_with_non_observable_dropna(self, dropna): name="A", ), ) + if not dropna: + expected["B"] = expected["B"].astype(float) tm.assert_frame_equal(result, expected) @@ -287,6 +289,8 @@ def test_pivot_with_interval_index(self, interval_values, dropna): df = DataFrame({"A": interval_values, "B": 1}) result = df.pivot_table(index="A", values="B", dropna=dropna) expected = DataFrame({"B": 1}, index=Index(interval_values.unique(), name="A")) + if not dropna: + expected = expected.astype(float) tm.assert_frame_equal(result, expected) def test_pivot_with_interval_index_margins(self): @@ -389,7 +393,7 @@ def test_pivot_preserve_dtypes(self, columns, values): result = dict(df_res.dtypes) expected = { - col: np.dtype("O") if col[0].startswith("b") else np.dtype("float64") + col: np.dtype("float64") for col in df_res } assert result == expected @@ -1712,8 +1716,9 @@ def test_pivot_table_margins_name_with_aggfunc_list(self): expected = DataFrame(table.values, index=ix, columns=cols) tm.assert_frame_equal(table, expected) - @pytest.mark.xfail(reason="GH#17035 (np.mean of ints is casted back to ints)") - def test_categorical_margins(self, observed): + def test_categorical_margins(self, observed, request): + if observed: + request.node.add_marker(pytest.mark.xfail(reason="GH#17035 (np.mean of ints is casted back to ints)")) # GH 10989 df = DataFrame( {"x": np.arange(8), "y": np.arange(8) // 4, "z": np.arange(8) % 2} @@ -1726,8 +1731,9 @@ def test_categorical_margins(self, observed): table = df.pivot_table("x", "y", "z", dropna=observed, margins=True) tm.assert_frame_equal(table, expected) - @pytest.mark.xfail(reason="GH#17035 (np.mean of ints is casted back to ints)") - def test_categorical_margins_category(self, observed): + def test_categorical_margins_category(self, observed, request): + if observed: + request.node.add_marker(pytest.mark.xfail(reason="GH#17035 (np.mean of ints is casted back to ints)")) df = DataFrame( {"x": np.arange(8), "y": np.arange(8) // 4, "z": np.arange(8) % 2} ) From eb572587a5dfaef6d5e45ae54070bb52c08ac0c4 Mon Sep 17 00:00:00 2001 From: Richard Shadrach Date: Wed, 31 Mar 2021 20:42:15 -0400 Subject: [PATCH 4/8] BUG: groupby sum, mean, var should always be floats --- doc/source/whatsnew/v1.3.0.rst | 30 +++++++++ pandas/core/groupby/generic.py | 9 +-- pandas/core/groupby/groupby.py | 13 ++-- pandas/core/groupby/ops.py | 11 ++-- pandas/tests/extension/base/groupby.py | 4 +- pandas/tests/extension/test_boolean.py | 4 +- .../tests/groupby/aggregate/test_aggregate.py | 66 +++++++++++++++++-- pandas/tests/groupby/test_categorical.py | 14 ++-- pandas/tests/groupby/test_function.py | 5 +- pandas/tests/groupby/test_groupby.py | 11 ++-- .../tests/groupby/transform/test_transform.py | 8 +-- pandas/tests/io/formats/test_to_csv.py | 2 +- pandas/tests/resample/test_datetime_index.py | 15 +++-- pandas/tests/resample/test_period_index.py | 12 ++-- .../tests/resample/test_resampler_grouper.py | 4 +- pandas/tests/resample/test_timedelta.py | 4 +- pandas/tests/reshape/test_crosstab.py | 2 + pandas/tests/reshape/test_pivot.py | 32 +++++---- 18 files changed, 175 insertions(+), 71 deletions(-) diff --git a/doc/source/whatsnew/v1.3.0.rst b/doc/source/whatsnew/v1.3.0.rst index 842b50ce53b21..f4ec6a92b954e 100644 --- a/doc/source/whatsnew/v1.3.0.rst +++ b/doc/source/whatsnew/v1.3.0.rst @@ -298,6 +298,36 @@ Preserve dtypes in :meth:`~pandas.DataFrame.combine_first` combined.dtypes +Group by methods agg and transform no longer changes return dtype for callables +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Previously the methods :meth:`.DataFrameGroupBy.aggregate`, +:meth:`.SeriesGroupBy.aggregate`, :meth:`.DataFrameGroupBy.transform`, and +:meth:`.SeriesGroupBy.transform` might cast the result dtype when the argument ``func`` +is callable, possibly leading to undesirable results (:issue:`21240`). The cast would +occur if the result is numeric and casting back to the input dtype does not change any +values as measured by ``np.allclose``. Now no such casting occurs. + +.. ipython:: python + + df = pd.DataFrame({'key': [1, 1], 'a': [True, False], 'b': [True, True]}) + df + +*pandas 1.2.x* + +.. code-block:: ipython + + In [5]: df.groupby('key').agg(lambda x: x.sum()) + Out[5]: + a b + key + 1 True 2 + +*pandas 1.3.0* + +.. ipython:: python + + In [5]: df.groupby('key').agg(lambda x: x.sum()) Try operating inplace when setting values with ``loc`` and ``iloc`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/pandas/core/groupby/generic.py b/pandas/core/groupby/generic.py index 9d6d2d698dfe5..46b71ac771c20 100644 --- a/pandas/core/groupby/generic.py +++ b/pandas/core/groupby/generic.py @@ -45,7 +45,7 @@ ) from pandas.core.dtypes.cast import ( - find_common_type, + maybe_cast_result_dtype, maybe_downcast_numeric, ) from pandas.core.dtypes.common import ( @@ -588,8 +588,9 @@ def transform(self, func, *args, engine=None, engine_kwargs=None, **kwargs): def _transform_general(self, func, *args, **kwargs): """ - Transform with a non-str `func`. + Transform with a callable func`. """ + assert callable(func) klass = type(self._selected_obj) results = [] @@ -613,10 +614,6 @@ def _transform_general(self, func, *args, **kwargs): # we will only try to coerce the result type if # we have a numeric dtype, as these are *always* user-defined funcs # the cython take a different path (and casting) - if is_numeric_dtype(result.dtype): - common_dtype = find_common_type([self._selected_obj.dtype, result.dtype]) - if common_dtype is result.dtype: - result = maybe_downcast_numeric(result, self._selected_obj.dtype) result.name = self._selected_obj.name return result diff --git a/pandas/core/groupby/groupby.py b/pandas/core/groupby/groupby.py index f579b04db898e..077b4a53103db 100644 --- a/pandas/core/groupby/groupby.py +++ b/pandas/core/groupby/groupby.py @@ -1241,9 +1241,6 @@ def _python_agg_general(self, func, *args, **kwargs): assert result is not None key = base.OutputKey(label=name, position=idx) - if is_numeric_dtype(obj.dtype): - result = maybe_downcast_numeric(result, obj.dtype) - if self.grouper._filter_empty_groups: mask = counts.ravel() > 0 @@ -1525,12 +1522,12 @@ def mean(self, numeric_only: bool = True): Groupby two columns and return the mean of the remaining column. >>> df.groupby(['A', 'B']).mean() - C + C A B - 1 2.0 2 - 4.0 1 - 2 3.0 1 - 5.0 2 + 1 2.0 2.0 + 4.0 1.0 + 2 3.0 1.0 + 5.0 2.0 Groupby one column and return the mean of only particular column in the group. diff --git a/pandas/core/groupby/ops.py b/pandas/core/groupby/ops.py index 3ee185d862b01..8c3b7eeba5571 100644 --- a/pandas/core/groupby/ops.py +++ b/pandas/core/groupby/ops.py @@ -290,10 +290,13 @@ def get_result_dtype(self, dtype: DtypeObj) -> DtypeObj: return np.dtype(np.int64) elif isinstance(dtype, (BooleanDtype, _IntegerDtype)): return Int64Dtype() - elif how in ["mean", "median", "var"] and isinstance( - dtype, (BooleanDtype, _IntegerDtype) - ): - return Float64Dtype() + elif how in ["mean", "median", "var"]: + if isinstance(dtype, (BooleanDtype, _IntegerDtype)): + return Float64Dtype() + elif is_float_dtype(dtype): + return dtype + elif is_numeric_dtype(dtype): + return np.dtype(np.float64) return dtype def uses_mask(self) -> bool: diff --git a/pandas/tests/extension/base/groupby.py b/pandas/tests/extension/base/groupby.py index 30b115b9dba6f..dbc519dec1b77 100644 --- a/pandas/tests/extension/base/groupby.py +++ b/pandas/tests/extension/base/groupby.py @@ -25,7 +25,7 @@ def test_groupby_extension_agg(self, as_index, data_for_grouping): _, index = pd.factorize(data_for_grouping, sort=True) index = pd.Index(index, name="B") - expected = pd.Series([3, 1, 4], index=index, name="A") + expected = pd.Series([3.0, 1.0, 4.0], index=index, name="A") if as_index: self.assert_series_equal(result, expected) else: @@ -54,7 +54,7 @@ def test_groupby_extension_no_sort(self, data_for_grouping): _, index = pd.factorize(data_for_grouping, sort=False) index = pd.Index(index, name="B") - expected = pd.Series([1, 3, 4], index=index, name="A") + expected = pd.Series([1.0, 3.0, 4.0], index=index, name="A") self.assert_series_equal(result, expected) def test_groupby_extension_transform(self, data_for_grouping): diff --git a/pandas/tests/extension/test_boolean.py b/pandas/tests/extension/test_boolean.py index 33d82a1d64fb7..8e61b4f4b9bb9 100644 --- a/pandas/tests/extension/test_boolean.py +++ b/pandas/tests/extension/test_boolean.py @@ -272,7 +272,7 @@ def test_groupby_extension_agg(self, as_index, data_for_grouping): _, index = pd.factorize(data_for_grouping, sort=True) index = pd.Index(index, name="B") - expected = pd.Series([3, 1], index=index, name="A") + expected = pd.Series([3.0, 1.0], index=index, name="A") if as_index: self.assert_series_equal(result, expected) else: @@ -301,7 +301,7 @@ def test_groupby_extension_no_sort(self, data_for_grouping): _, index = pd.factorize(data_for_grouping, sort=False) index = pd.Index(index, name="B") - expected = pd.Series([1, 3], index=index, name="A") + expected = pd.Series([1.0, 3.0], index=index, name="A") self.assert_series_equal(result, expected) def test_groupby_extension_transform(self, data_for_grouping): diff --git a/pandas/tests/groupby/aggregate/test_aggregate.py b/pandas/tests/groupby/aggregate/test_aggregate.py index 28344897a686f..0247ab52f1e9f 100644 --- a/pandas/tests/groupby/aggregate/test_aggregate.py +++ b/pandas/tests/groupby/aggregate/test_aggregate.py @@ -234,11 +234,10 @@ def test_aggregate_item_by_item(df): K = len(result.columns) # GH5782 - # odd comparisons can result here, so cast to make easy - exp = Series(np.array([foo] * K), index=list("BCD"), dtype=np.float64, name="foo") + exp = Series(np.array([foo] * K), index=list("BCD"), name="foo") tm.assert_series_equal(result.xs("foo"), exp) - exp = Series(np.array([bar] * K), index=list("BCD"), dtype=np.float64, name="bar") + exp = Series(np.array([bar] * K), index=list("BCD"), name="bar") tm.assert_almost_equal(result.xs("bar"), exp) def aggfun(ser): @@ -442,6 +441,57 @@ def test_bool_agg_dtype(op): assert is_integer_dtype(result) +@pytest.mark.parametrize( + "keys, agg_index", + [ + (["a"], Index([1], name="a")), + (["a", "b"], MultiIndex([[1], [2]], [[0], [0]], names=["a", "b"])), + ], +) +@pytest.mark.parametrize( + "input_dtype", ["bool", "int32", "int64", "float32", "float64"] +) +@pytest.mark.parametrize( + "result_dtype", ["bool", "int32", "int64", "float32", "float64"] +) +@pytest.mark.parametrize("method", ["apply", "aggregate", "transform"]) +def test_callable_result_dtype_frame( + keys, agg_index, input_dtype, result_dtype, method +): + # GH 21240 + df = DataFrame({"a": [1], "b": [2], "c": [True]}) + df["c"] = df["c"].astype(input_dtype) + op = getattr(df.groupby(keys)[["c"]], method) + result = op(lambda x: x.astype(result_dtype).iloc[0]) + expected_index = pd.RangeIndex(0, 1) if method == "transform" else agg_index + expected = DataFrame({"c": [df["c"].iloc[0]]}, index=expected_index).astype( + result_dtype + ) + if method == "apply": + expected.columns.names = [0] + tm.assert_frame_equal(result, expected) + + +@pytest.mark.parametrize( + "keys, agg_index", + [ + (["a"], Index([1], name="a")), + (["a", "b"], MultiIndex([[1], [2]], [[0], [0]], names=["a", "b"])), + ], +) +@pytest.mark.parametrize("input", [True, 1, 1.0]) +@pytest.mark.parametrize("dtype", [bool, int, float]) +@pytest.mark.parametrize("method", ["apply", "aggregate", "transform"]) +def test_callable_result_dtype_series(keys, agg_index, input, dtype, method): + # GH 21240 + df = DataFrame({"a": [1], "b": [2], "c": [input]}) + op = getattr(df.groupby(keys)["c"], method) + result = op(lambda x: x.astype(dtype).iloc[0]) + expected_index = pd.RangeIndex(0, 1) if method == "transform" else agg_index + expected = Series([df["c"].iloc[0]], index=expected_index, name="c").astype(dtype) + tm.assert_series_equal(result, expected) + + def test_order_aggregate_multiple_funcs(): # GH 25692 df = DataFrame({"A": [1, 1, 2, 2], "B": [1, 2, 3, 4]}) @@ -462,7 +512,9 @@ def test_uint64_type_handling(dtype, how): expected = df.groupby("y").agg({"x": how}) df.x = df.x.astype(dtype) result = df.groupby("y").agg({"x": how}) - result.x = result.x.astype(np.int64) + if how not in ("mean", "median"): + # mean and median always result in floats + result.x = result.x.astype(np.int64) tm.assert_frame_equal(result, expected, check_exact=True) @@ -849,7 +901,11 @@ def test_multiindex_custom_func(func): data = [[1, 4, 2], [5, 7, 1]] df = DataFrame(data, columns=MultiIndex.from_arrays([[1, 1, 2], [3, 4, 3]])) result = df.groupby(np.array([0, 1])).agg(func) - expected_dict = {(1, 3): {0: 1, 1: 5}, (1, 4): {0: 4, 1: 7}, (2, 3): {0: 2, 1: 1}} + expected_dict = { + (1, 3): {0: 1.0, 1: 5.0}, + (1, 4): {0: 4.0, 1: 7.0}, + (2, 3): {0: 2.0, 1: 1.0}, + } expected = DataFrame(expected_dict) tm.assert_frame_equal(result, expected) diff --git a/pandas/tests/groupby/test_categorical.py b/pandas/tests/groupby/test_categorical.py index da438826a939a..d685fb4e5eb51 100644 --- a/pandas/tests/groupby/test_categorical.py +++ b/pandas/tests/groupby/test_categorical.py @@ -285,8 +285,6 @@ def test_apply(ordered): result = grouped.apply(lambda x: np.mean(x)) tm.assert_frame_equal(result, expected) - # we coerce back to ints - expected = expected.astype("int") result = grouped.mean() tm.assert_frame_equal(result, expected) @@ -371,7 +369,7 @@ def test_observed(observed, using_array_manager): result = groups_double_key.agg("mean") expected = DataFrame( { - "val": [10, 30, 20, 40], + "val": [10.0, 30, 20, 40], "cat": Categorical( ["a", "a", "b", "b"], categories=["a", "b", "c"], ordered=True ), @@ -418,7 +416,9 @@ def test_observed_codes_remap(observed): groups_double_key = df.groupby([values, "C2"], observed=observed) idx = MultiIndex.from_arrays([values, [1, 2, 3, 4]], names=["cat", "C2"]) - expected = DataFrame({"C1": [3, 3, 4, 5], "C3": [10, 100, 200, 34]}, index=idx) + expected = DataFrame( + {"C1": [3.0, 3.0, 4.0, 5.0], "C3": [10.0, 100.0, 200.0, 34.0]}, index=idx + ) if not observed: expected = cartesian_product_for_groupers( expected, [values.values, [1, 2, 3, 4]], ["cat", "C2"] @@ -1505,7 +1505,9 @@ def test_read_only_category_no_sort(): df = DataFrame( {"a": [1, 3, 5, 7], "b": Categorical([1, 1, 2, 2], categories=Index(cats))} ) - expected = DataFrame(data={"a": [2, 6]}, index=CategoricalIndex([1, 2], name="b")) + expected = DataFrame( + data={"a": [2.0, 6.0]}, index=CategoricalIndex([1, 2], name="b") + ) result = df.groupby("b", sort=False).mean() tm.assert_frame_equal(result, expected) @@ -1597,7 +1599,7 @@ def test_aggregate_categorical_with_isnan(): index = MultiIndex.from_arrays([[1, 1], [1, 2]], names=("A", "B")) expected = DataFrame( data={ - "numerical_col": [1.0, 0.0], + "numerical_col": [1, 0], "object_col": [0, 0], "categorical_col": [0, 0], }, diff --git a/pandas/tests/groupby/test_function.py b/pandas/tests/groupby/test_function.py index f47fc1f4e4a4f..3c620b59dbc2a 100644 --- a/pandas/tests/groupby/test_function.py +++ b/pandas/tests/groupby/test_function.py @@ -408,7 +408,8 @@ def test_median_empty_bins(observed): result = df.groupby(bins, observed=observed).median() expected = df.groupby(bins, observed=observed).agg(lambda x: x.median()) - tm.assert_frame_equal(result, expected) + # TODO: GH 41137 + tm.assert_frame_equal(result, expected, check_dtype=False) @pytest.mark.parametrize( @@ -588,7 +589,7 @@ def test_ops_general(op, targop): df = DataFrame(np.random.randn(1000)) labels = np.random.randint(0, 50, size=1000).astype(float) - result = getattr(df.groupby(labels), op)().astype(float) + result = getattr(df.groupby(labels), op)() expected = df.groupby(labels).agg(targop) tm.assert_frame_equal(result, expected) diff --git a/pandas/tests/groupby/test_groupby.py b/pandas/tests/groupby/test_groupby.py index 3f6485be871f1..b14bbfa7d514d 100644 --- a/pandas/tests/groupby/test_groupby.py +++ b/pandas/tests/groupby/test_groupby.py @@ -299,10 +299,9 @@ def f(x): return float(len(x)) agged = grouped.agg(f) - expected = Series([4, 2], index=["bar", "foo"]) - tm.assert_series_equal(agged, expected, check_dtype=False) - assert issubclass(agged.dtype.type, np.dtype(dtype).type) + expected = Series([4.0, 2.0], index=["bar", "foo"]) + tm.assert_series_equal(agged, expected) def test_indices_concatenation_order(): @@ -1237,7 +1236,7 @@ def test_groupby_keys_same_size_as_index(): ) df = DataFrame([["A", 10], ["B", 15]], columns=["metric", "values"], index=index) result = df.groupby([Grouper(level=0, freq=freq), "metric"]).mean() - expected = df.set_index([df.index, "metric"]) + expected = df.set_index([df.index, "metric"]).astype(float) tm.assert_frame_equal(result, expected) @@ -1330,7 +1329,7 @@ def test_groupby_2d_malformed(): d["ones"] = [1, 1] d["label"] = ["l1", "l2"] tmp = d.groupby(["group"]).mean() - res_values = np.array([[0, 1], [0, 1]], dtype=np.int64) + res_values = np.array([[0.0, 1.0], [0.0, 1.0]]) tm.assert_index_equal(tmp.columns, Index(["zeros", "ones"])) tm.assert_numpy_array_equal(tmp.values, res_values) @@ -2023,7 +2022,7 @@ def test_groupby_crash_on_nunique(axis): def test_groupby_list_level(): # GH 9790 - expected = DataFrame(np.arange(0, 9).reshape(3, 3)) + expected = DataFrame(np.arange(0, 9).reshape(3, 3), dtype=float) result = expected.groupby(level=[0]).mean() tm.assert_frame_equal(result, expected) diff --git a/pandas/tests/groupby/transform/test_transform.py b/pandas/tests/groupby/transform/test_transform.py index 9350a3fcd3036..aa0f90e83ab5b 100644 --- a/pandas/tests/groupby/transform/test_transform.py +++ b/pandas/tests/groupby/transform/test_transform.py @@ -242,7 +242,7 @@ def test_transform_bug(): # transforming on a datetime column df = DataFrame({"A": Timestamp("20130101"), "B": np.arange(5)}) result = df.groupby("A")["B"].transform(lambda x: x.rank(ascending=False)) - expected = Series(np.arange(5, 0, step=-1), name="B") + expected = Series(np.arange(5, 0, step=-1), name="B", dtype="float64") tm.assert_series_equal(result, expected) @@ -493,7 +493,7 @@ def test_groupby_transform_with_int(): ) with np.errstate(all="ignore"): result = df.groupby("A").transform(lambda x: (x - x.mean()) / x.std()) - expected = DataFrame({"B": np.nan, "C": [-1, 0, 1, -1, 0, 1]}) + expected = DataFrame({"B": np.nan, "C": [-1.0, 0.0, 1.0, -1.0, 0.0, 1.0]}) tm.assert_frame_equal(result, expected) # int that needs float conversion @@ -509,9 +509,9 @@ def test_groupby_transform_with_int(): expected = DataFrame({"B": np.nan, "C": concat([s1, s2])}) tm.assert_frame_equal(result, expected) - # int downcasting + # int doesn't get downcasted result = df.groupby("A").transform(lambda x: x * 2 / 2) - expected = DataFrame({"B": 1, "C": [2, 3, 4, 10, 5, -1]}) + expected = DataFrame({"B": 1.0, "C": [2.0, 3.0, 4.0, 10.0, 5.0, -1.0]}) tm.assert_frame_equal(result, expected) diff --git a/pandas/tests/io/formats/test_to_csv.py b/pandas/tests/io/formats/test_to_csv.py index bbf78a9013731..4c482bafa6c9c 100644 --- a/pandas/tests/io/formats/test_to_csv.py +++ b/pandas/tests/io/formats/test_to_csv.py @@ -274,7 +274,7 @@ def test_to_csv_date_format(self): df_sec["B"] = 0 df_sec["C"] = 1 - expected_rows = ["A,B,C", "2013-01-01,0,1"] + expected_rows = ["A,B,C", "2013-01-01,0,1.0"] expected_ymd_sec = tm.convert_rows_list_to_csv_str(expected_rows) df_sec_grouped = df_sec.groupby([pd.Grouper(key="A", freq="1h"), "B"]) diff --git a/pandas/tests/resample/test_datetime_index.py b/pandas/tests/resample/test_datetime_index.py index bbe9ac6fa8094..8b56e9379895f 100644 --- a/pandas/tests/resample/test_datetime_index.py +++ b/pandas/tests/resample/test_datetime_index.py @@ -1064,7 +1064,7 @@ def test_nanosecond_resample_error(): result = r.agg("mean") exp_indx = date_range(start=pd.to_datetime(exp_start), periods=10, freq="100n") - exp = Series(range(len(exp_indx)), index=exp_indx) + exp = Series(range(len(exp_indx)), index=exp_indx, dtype=float) tm.assert_series_equal(result, exp) @@ -1206,6 +1206,9 @@ def test_resample_median_bug_1688(): result = df.resample("T").apply(lambda x: x.mean()) exp = df.asfreq("T") + if dtype == "float32": + # TODO: Empty groups cause x.mean() to return float64 + exp = exp.astype("float64") tm.assert_frame_equal(result, exp) result = df.resample("T").median() @@ -1633,15 +1636,15 @@ def test_resample_with_nat(): index_1s = DatetimeIndex( ["1970-01-01 00:00:00", "1970-01-01 00:00:01", "1970-01-01 00:00:02"] ) - frame_1s = DataFrame([3, 7, 11], index=index_1s) + frame_1s = DataFrame([3.0, 7.0, 11.0], index=index_1s) tm.assert_frame_equal(frame.resample("1s").mean(), frame_1s) index_2s = DatetimeIndex(["1970-01-01 00:00:00", "1970-01-01 00:00:02"]) - frame_2s = DataFrame([5, 11], index=index_2s) + frame_2s = DataFrame([5.0, 11.0], index=index_2s) tm.assert_frame_equal(frame.resample("2s").mean(), frame_2s) index_3s = DatetimeIndex(["1970-01-01 00:00:00"]) - frame_3s = DataFrame([7], index=index_3s) + frame_3s = DataFrame([7.0], index=index_3s) tm.assert_frame_equal(frame.resample("3s").mean(), frame_3s) tm.assert_frame_equal(frame.resample("60s").mean(), frame_3s) @@ -1684,8 +1687,10 @@ def f(data, add_arg): # Testing dataframe df = DataFrame({"A": 1, "B": 2}, index=date_range("2017", periods=10)) - result = df.groupby("A").resample("D").agg(f, multiplier) + result = df.groupby("A").resample("D").agg(f, multiplier).astype(float) expected = df.groupby("A").resample("D").mean().multiply(multiplier) + # TODO: GH 41137 + expected = expected.astype("float64") tm.assert_frame_equal(result, expected) diff --git a/pandas/tests/resample/test_period_index.py b/pandas/tests/resample/test_period_index.py index e2b13f6a00677..a6491952375a4 100644 --- a/pandas/tests/resample/test_period_index.py +++ b/pandas/tests/resample/test_period_index.py @@ -269,7 +269,7 @@ def test_with_local_timezone_pytz(self): # Index is moved back a day with the timezone conversion from UTC to # Pacific expected_index = period_range(start=start, end=end, freq="D") - offsets.Day() - expected = Series(1, index=expected_index) + expected = Series(1.0, index=expected_index) tm.assert_series_equal(result, expected) def test_resample_with_pytz(self): @@ -279,7 +279,7 @@ def test_resample_with_pytz(self): ) result = s.resample("D").mean() expected = Series( - 2, + 2.0, index=pd.DatetimeIndex( ["2017-01-01", "2017-01-02"], tz="US/Eastern", freq="D" ), @@ -312,7 +312,7 @@ def test_with_local_timezone_dateutil(self): expected_index = ( period_range(start=start, end=end, freq="D", name="idx") - offsets.Day() ) - expected = Series(1, index=expected_index) + expected = Series(1.0, index=expected_index) tm.assert_series_equal(result, expected) def test_resample_nonexistent_time_bin_edge(self): @@ -777,8 +777,8 @@ def test_upsampling_ohlc(self, freq, period_mult, kind): "freq, expected_values", [ ("1s", [3, np.NaN, 7, 11]), - ("2s", [3, int((7 + 11) / 2)]), - ("3s", [int((3 + 7) / 2), 11]), + ("2s", [3, (7 + 11) / 2]), + ("3s", [(3 + 7) / 2, 11]), ], ) def test_resample_with_nat(self, periods, values, freq, expected_values): @@ -798,7 +798,7 @@ def test_resample_with_only_nat(self): pi = PeriodIndex([pd.NaT] * 3, freq="S") frame = DataFrame([2, 3, 5], index=pi, columns=["a"]) expected_index = PeriodIndex(data=[], freq=pi.freq) - expected = DataFrame(index=expected_index, columns=["a"], dtype="int64") + expected = DataFrame(index=expected_index, columns=["a"], dtype="float64") result = frame.resample("1s").mean() tm.assert_frame_equal(result, expected) diff --git a/pandas/tests/resample/test_resampler_grouper.py b/pandas/tests/resample/test_resampler_grouper.py index 999d8a6c90ba2..0fa9ad60f32e7 100644 --- a/pandas/tests/resample/test_resampler_grouper.py +++ b/pandas/tests/resample/test_resampler_grouper.py @@ -258,6 +258,8 @@ def f(x): return x.resample("2s").apply(lambda y: y.sum()) result = g.apply(f) + # y.sum() results in int64 instead of int32 on 32-bit architectures + expected = expected.astype("int64") tm.assert_frame_equal(result, expected) @@ -289,7 +291,7 @@ def test_apply_columns_multilevel(): agg_dict = {col: (np.sum if col[3] == "one" else np.mean) for col in df.columns} result = df.resample("H").apply(lambda x: agg_dict[x.name](x)) expected = DataFrame( - np.array([0] * 4).reshape(2, 2), + 2 * [[0, 0.0]], index=date_range(start="2017-01-01", freq="1H", periods=2), columns=pd.MultiIndex.from_tuples( [("A", "a", "", "one"), ("B", "b", "i", "two")] diff --git a/pandas/tests/resample/test_timedelta.py b/pandas/tests/resample/test_timedelta.py index c6ee295208607..6e2d2fd55d89c 100644 --- a/pandas/tests/resample/test_timedelta.py +++ b/pandas/tests/resample/test_timedelta.py @@ -77,7 +77,7 @@ def test_resample_timedelta_idempotency(): index = timedelta_range("0", periods=9, freq="10L") series = Series(range(9), index=index) result = series.resample("10L").mean() - expected = series + expected = series.astype(float) tm.assert_series_equal(result, expected) @@ -162,7 +162,7 @@ def test_resample_with_timedelta_yields_no_empty_groups(): result = df.loc["1s":, :].resample("3s").apply(lambda x: len(x)) expected = DataFrame( - [[768.0] * 4] * 12 + [[528.0] * 4], + [[768] * 4] * 12 + [[528] * 4], index=timedelta_range(start="1s", periods=13, freq="3s"), ) tm.assert_frame_equal(result, expected) diff --git a/pandas/tests/reshape/test_crosstab.py b/pandas/tests/reshape/test_crosstab.py index 44299d51a878f..62fd93026d5e2 100644 --- a/pandas/tests/reshape/test_crosstab.py +++ b/pandas/tests/reshape/test_crosstab.py @@ -559,6 +559,8 @@ def test_crosstab_with_numpy_size(self): expected = DataFrame( expected_data, index=expected_index, columns=expected_column ) + # aggfunc is np.size, resulting in integers + expected["All"] = expected["All"].astype("int64") tm.assert_frame_equal(result, expected) def test_crosstab_duplicate_names(self): diff --git a/pandas/tests/reshape/test_pivot.py b/pandas/tests/reshape/test_pivot.py index 3d1c3b81c492f..63e277c1580af 100644 --- a/pandas/tests/reshape/test_pivot.py +++ b/pandas/tests/reshape/test_pivot.py @@ -240,13 +240,13 @@ def test_pivot_with_non_observable_dropna(self, dropna): categories=["low", "high"], ordered=True, ), - "B": range(5), + "B": [0.0, 1.0, 2.0, 3.0, 4.0], } ) result = df.pivot_table(index="A", values="B", dropna=dropna) expected = DataFrame( - {"B": [2, 3]}, + {"B": [2.0, 3.0]}, index=Index( Categorical.from_codes( [0, 1], categories=["low", "high"], ordered=True @@ -279,6 +279,8 @@ def test_pivot_with_non_observable_dropna(self, dropna): name="A", ), ) + if not dropna: + expected["B"] = expected["B"].astype(float) tm.assert_frame_equal(result, expected) @@ -287,6 +289,8 @@ def test_pivot_with_interval_index(self, interval_values, dropna): df = DataFrame({"A": interval_values, "B": 1}) result = df.pivot_table(index="A", values="B", dropna=dropna) expected = DataFrame({"B": 1}, index=Index(interval_values.unique(), name="A")) + if not dropna: + expected = expected.astype(float) tm.assert_frame_equal(result, expected) def test_pivot_with_interval_index_margins(self): @@ -388,10 +392,7 @@ def test_pivot_preserve_dtypes(self, columns, values): ) result = dict(df_res.dtypes) - expected = { - col: np.dtype("O") if col[0].startswith("b") else np.dtype("float64") - for col in df_res - } + expected = {col: np.dtype("float64") for col in df_res} assert result == expected def test_pivot_no_values(self): @@ -986,7 +987,6 @@ def test_margins_dtype(self): tm.assert_frame_equal(expected, result) - @pytest.mark.xfail(reason="GH#17035 (len of floats is casted back to floats)") def test_margins_dtype_len(self): mi_val = list(product(["bar", "foo"], ["one", "two"])) + [("All", "")] mi = MultiIndex.from_tuples(mi_val, names=("A", "B")) @@ -1712,8 +1712,13 @@ def test_pivot_table_margins_name_with_aggfunc_list(self): expected = DataFrame(table.values, index=ix, columns=cols) tm.assert_frame_equal(table, expected) - @pytest.mark.xfail(reason="GH#17035 (np.mean of ints is casted back to ints)") - def test_categorical_margins(self, observed): + def test_categorical_margins(self, observed, request): + if observed: + request.node.add_marker( + pytest.mark.xfail( + reason="GH#17035 (np.mean of ints is casted back to ints)" + ) + ) # GH 10989 df = DataFrame( {"x": np.arange(8), "y": np.arange(8) // 4, "z": np.arange(8) % 2} @@ -1726,8 +1731,13 @@ def test_categorical_margins(self, observed): table = df.pivot_table("x", "y", "z", dropna=observed, margins=True) tm.assert_frame_equal(table, expected) - @pytest.mark.xfail(reason="GH#17035 (np.mean of ints is casted back to ints)") - def test_categorical_margins_category(self, observed): + def test_categorical_margins_category(self, observed, request): + if observed: + request.node.add_marker( + pytest.mark.xfail( + reason="GH#17035 (np.mean of ints is casted back to ints)" + ) + ) df = DataFrame( {"x": np.arange(8), "y": np.arange(8) // 4, "z": np.arange(8) % 2} ) From 5be32536a11a417d9828b738bf02bdca5b0bba62 Mon Sep 17 00:00:00 2001 From: Richard Shadrach Date: Sat, 24 Apr 2021 17:36:49 -0400 Subject: [PATCH 5/8] Fixups --- pandas/core/groupby/generic.py | 4 ---- pandas/tests/groupby/aggregate/test_cython.py | 2 -- pandas/tests/resample/test_datetime_index.py | 2 -- 3 files changed, 8 deletions(-) diff --git a/pandas/core/groupby/generic.py b/pandas/core/groupby/generic.py index 46b71ac771c20..70801651b5aef 100644 --- a/pandas/core/groupby/generic.py +++ b/pandas/core/groupby/generic.py @@ -44,10 +44,6 @@ doc, ) -from pandas.core.dtypes.cast import ( - maybe_cast_result_dtype, - maybe_downcast_numeric, -) from pandas.core.dtypes.common import ( ensure_int64, is_bool, diff --git a/pandas/tests/groupby/aggregate/test_cython.py b/pandas/tests/groupby/aggregate/test_cython.py index 5aa0bf413a9df..4a8aabe41b754 100644 --- a/pandas/tests/groupby/aggregate/test_cython.py +++ b/pandas/tests/groupby/aggregate/test_cython.py @@ -196,8 +196,6 @@ def test_cython_agg_empty_buckets(op, targop, observed): g = df.groupby(pd.cut(df[0], grps), observed=observed) expected = g.agg(lambda x: targop(x)) - if op in ("mean", "median", "var"): - expected = expected.astype(float) tm.assert_frame_equal(result, expected) diff --git a/pandas/tests/resample/test_datetime_index.py b/pandas/tests/resample/test_datetime_index.py index 8b56e9379895f..6f2e3d9505152 100644 --- a/pandas/tests/resample/test_datetime_index.py +++ b/pandas/tests/resample/test_datetime_index.py @@ -1689,8 +1689,6 @@ def f(data, add_arg): df = DataFrame({"A": 1, "B": 2}, index=date_range("2017", periods=10)) result = df.groupby("A").resample("D").agg(f, multiplier).astype(float) expected = df.groupby("A").resample("D").mean().multiply(multiplier) - # TODO: GH 41137 - expected = expected.astype("float64") tm.assert_frame_equal(result, expected) From 43ab5ca4da3143b09192f3a34b7b43c7ab2f9d96 Mon Sep 17 00:00:00 2001 From: Richard Shadrach Date: Sat, 24 Apr 2021 22:29:26 -0400 Subject: [PATCH 6/8] Update docstring --- pandas/core/groupby/grouper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/core/groupby/grouper.py b/pandas/core/groupby/grouper.py index 151756b829a1d..d099b7884c84c 100644 --- a/pandas/core/groupby/grouper.py +++ b/pandas/core/groupby/grouper.py @@ -150,8 +150,8 @@ class Grouper: >>> df.groupby(pd.Grouper(key="Animal")).mean() Speed Animal - Falcon 200 - Parrot 10 + Falcon 200.0 + Parrot 10.0 Specify a resample operation on the column 'Publish date' From bcd369c57a184570a77c9aff48bd4282e1b1ffbc Mon Sep 17 00:00:00 2001 From: Richard Shadrach Date: Wed, 5 May 2021 23:44:41 -0400 Subject: [PATCH 7/8] merge cleanup --- pandas/tests/groupby/aggregate/test_cython.py | 3 --- pandas/tests/groupby/test_groupby.py | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/pandas/tests/groupby/aggregate/test_cython.py b/pandas/tests/groupby/aggregate/test_cython.py index ded10ab11d5a8..4a8aabe41b754 100644 --- a/pandas/tests/groupby/aggregate/test_cython.py +++ b/pandas/tests/groupby/aggregate/test_cython.py @@ -196,9 +196,6 @@ def test_cython_agg_empty_buckets(op, targop, observed): g = df.groupby(pd.cut(df[0], grps), observed=observed) expected = g.agg(lambda x: targop(x)) - if observed and op not in ("min", "max"): - # TODO: GH 41137 - expected = expected.astype("int64") tm.assert_frame_equal(result, expected) diff --git a/pandas/tests/groupby/test_groupby.py b/pandas/tests/groupby/test_groupby.py index 96a59cc428814..ac37b75888205 100644 --- a/pandas/tests/groupby/test_groupby.py +++ b/pandas/tests/groupby/test_groupby.py @@ -1236,7 +1236,7 @@ def test_groupby_keys_same_size_as_index(): ) df = DataFrame([["A", 10], ["B", 15]], columns=["metric", "values"], index=index) result = df.groupby([Grouper(level=0, freq=freq), "metric"]).mean() - expected = df.set_index([df.index, "metric"]) + expected = df.set_index([df.index, "metric"]).astype(float) tm.assert_frame_equal(result, expected) From b6de44a4ee3b4d4f2c87ab4df3c89fdac76d4e86 Mon Sep 17 00:00:00 2001 From: Richard Shadrach Date: Sat, 8 May 2021 08:05:55 -0400 Subject: [PATCH 8/8] whatsnew --- doc/source/whatsnew/v1.3.0.rst | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/doc/source/whatsnew/v1.3.0.rst b/doc/source/whatsnew/v1.3.0.rst index cf3dd1b0e3226..bd567b2505da8 100644 --- a/doc/source/whatsnew/v1.3.0.rst +++ b/doc/source/whatsnew/v1.3.0.rst @@ -331,6 +331,31 @@ values as measured by ``np.allclose``. Now no such casting occurs. df.groupby('key').agg(lambda x: x.sum()) +``float`` result for :meth:`.GroupBy.mean`, :meth:`.GroupBy.median`, and :meth:`.GroupBy.var` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Previously, these methods could result in different dtypes depending on the input values. +Now, these methods will always return a float dtype. (:issue:`41137`) + +.. ipython:: python + + df = pd.DataFrame({'a': [True], 'b': [1], 'c': [1.0]}) + +*pandas 1.2.x* + +.. code-block:: ipython + + In [5]: df.groupby(df.index).mean() + Out[5]: + a b c + 0 True 1 1.0 + +*pandas 1.3.0* + +.. ipython:: python + + df.groupby(df.index).mean() + Try operating inplace when setting values with ``loc`` and ``iloc`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^