more wip

Author: MarcoGorelli

pandas/core/generic.py

@@ -9938,7 +9938,6 @@ def _add_numeric_operations(cls):
         )
         @Appender(_num_doc_mad)
         def mad(self, axis=None, skipna=None, level=None):
-            breakpoint()
             if skipna is None:
                 skipna = True
             if axis is None:

pandas/core/groupby/generic.py

@@ -927,7 +927,6 @@ def aggregate(self, func=None, *args, **kwargs):
             raise TypeError("Must provide 'func' or tuples of '(column, aggfunc).")
 
         func = maybe_mangle_lambdas(func)
-
         result, how = self._aggregate(func, *args, **kwargs)
         if how is None:
             return result
@@ -1131,7 +1130,6 @@ def _cython_agg_blocks(
     def _aggregate_frame(self, func, *args, **kwargs) -> DataFrame:
         if self.grouper.nkeys != 1:
             raise AssertionError("Number of keys must be 1")
-
         axis = self.axis
         obj = self._obj_with_exclusions
 
@@ -1145,7 +1143,6 @@ def _aggregate_frame(self, func, *args, **kwargs) -> DataFrame:
                 data = self.get_group(name, obj=obj)
                 fres = func(data, *args, **kwargs)
                 result[name] = fres
-
         return self._wrap_frame_output(result, obj)
 
     def _aggregate_item_by_item(self, func, *args, **kwargs) -> DataFrame:
@@ -1185,7 +1182,7 @@ def _wrap_applied_output(self, keys, values, not_indexed_same=False):
         if len(keys) == 0:
             return DataFrame(index=keys)
 
-        key_names = self.grouper.names
+        # key_names = self.grouper.names
 
         # GH12824.
         def first_not_none(values):
@@ -1195,35 +1192,37 @@ def first_not_none(values):
                 return None
 
         v = first_not_none(values)
-
         if v is None:
             # GH9684. If all values are None, then this will throw an error.
             # We'd prefer it return an empty dataframe.
             return DataFrame()
         elif isinstance(v, DataFrame):
             return self._concat_objects(keys, values, not_indexed_same=not_indexed_same)
         elif self.grouper.groupings is not None:
-            if len(self.grouper.groupings) > 1:
-                key_index = self.grouper.result_index
+            # if len(self.grouper.groupings) > 1:
+            key_index = self.grouper.result_index
+            if not self.as_index:
+                key_index = None
 
-            else:
-                ping = self.grouper.groupings[0]
-                if len(keys) == ping.ngroups:
-                    key_index = ping.group_index
-                    key_index.name = key_names[0]
+            # else:
+            #     breakpoint()
+            #     ping = self.grouper.groupings[0]
+            #     if len(keys) == ping.ngroups:
+            #         key_index = ping.result_index
+            #         key_index.name = key_names[0]
 
-                    key_lookup = Index(keys)
-                    indexer = key_lookup.get_indexer(key_index)
+            #         key_lookup = Index(keys)
+            #         indexer = key_lookup.get_indexer(key_index)
 
-                    # reorder the values
-                    values = [values[i] for i in indexer]
-                else:
+            #         # reorder the values
+            #         values = [values[i] for i in indexer]
+            #     else:
 
-                    key_index = Index(keys, name=key_names[0])
+            #         key_index = Index(keys, name=key_names[0])
 
-                # don't use the key indexer
-                if not self.as_index:
-                    key_index = None
+            #     # don't use the key indexer
+            #     if not self.as_index:
+            #         key_index = None
 
             # make Nones an empty object
             v = first_not_none(values)
@@ -1635,7 +1634,7 @@ def _gotitem(self, key, ndim: int, subset=None):
         raise AssertionError("invalid ndim for _gotitem")
 
     def _wrap_frame_output(self, result, obj) -> DataFrame:
-        result_index = self.grouper.levels[0]
+        result_index = self.grouper.result_index
 
         if self.axis == 0:
             return DataFrame(result, index=obj.columns, columns=result_index).T

pandas/core/groupby/groupby.py

@@ -641,6 +641,7 @@ def curried(x):
                 return self.apply(curried)
 
             try:
+                # breakpoint()
                 return self.apply(curried)
             except TypeError as err:
                 if not re.search(
@@ -728,10 +729,11 @@ def f(g):
                 )
         else:
             f = func
-
+        # breakpoint()
         # ignore SettingWithCopy here in case the user mutates
         with option_context("mode.chained_assignment", None):
             try:
+                # breakpoint()
                 result = self._python_apply_general(f)
             except TypeError:
                 # gh-20949
@@ -748,8 +750,9 @@ def f(g):
         return result
 
     def _python_apply_general(self, f):
+        # breakpoint()
         keys, values, mutated = self.grouper.apply(f, self._selected_obj, self.axis)
-
+        # breakpoint()
         return self._wrap_applied_output(
             keys, values, not_indexed_same=mutated or self.mutated
         )
@@ -943,7 +946,6 @@ def _python_agg_general(self, func, *args, **kwargs):
                     values = ensure_float(values)
 
                 output[key] = self._try_cast(values[mask], result)
-
         return self._wrap_aggregated_output(output)
 
     def _concat_objects(self, keys, values, not_indexed_same: bool = False):

pandas/core/groupby/ops.py

@@ -140,15 +140,21 @@ def _get_grouper(self):
         return self.groupings[0].grouper
 
     def _get_group_keys(self):
-        if len(self.groupings) == 1:
-            return self.levels[0]
-        else:
-            comp_ids, _, ngroups = self.group_info
+        # if len(self.groupings) == 1:
+        #     return self.levels[0]
+        # else:
+        comp_ids, _, ngroups = self.group_info
 
-            # provide "flattened" iterator for multi-group setting
-            return get_flattened_iterator(comp_ids, ngroups, self.levels, self.codes)
+        # provide "flattened" iterator for multi-group setting
+        flattened_iterator = get_flattened_iterator(
+            comp_ids, ngroups, self.levels, self.codes
+        )
+        if len(self.groupings) == 1:
+            return Index([i[0] for i in flattened_iterator], name=self.levels[0].name)
+        return flattened_iterator
 
     def apply(self, f, data: FrameOrSeries, axis: int = 0):
+        # breakpoint()
         mutated = self.mutated
         splitter = self._get_splitter(data, axis=axis)
         group_keys = self._get_group_keys()
@@ -261,6 +267,7 @@ def is_monotonic(self) -> bool:
 
     @cache_readonly
     def group_info(self):
+        # breakpoint()
         comp_ids, obs_group_ids = self._get_compressed_codes()
 
         ngroups = len(obs_group_ids)
@@ -278,6 +285,7 @@ def codes_info(self) -> np.ndarray:
 
     def _get_compressed_codes(self) -> Tuple[np.ndarray, np.ndarray]:
         all_codes = self.codes
+        # breakpoint()
         group_index = get_group_index(all_codes, self.shape, sort=True, xnull=True)
         return compress_group_index(group_index, sort=self.sort)
 
@@ -290,12 +298,14 @@ def ngroups(self) -> int:
 
     @property
     def reconstructed_codes(self) -> List[np.ndarray]:
+        # breakpoint()
         codes = self.codes
         comp_ids, obs_ids, _ = self.group_info
         return decons_obs_group_ids(comp_ids, obs_ids, self.shape, codes, xnull=True)
 
     @cache_readonly
     def result_index(self) -> Index:
+        # breakpoint()
         # if not self.compressed and len(self.groupings) == 1:
         #     return self.groupings[0].result_index.rename(self.names[0])
         codes = self.reconstructed_codes
@@ -304,6 +314,7 @@ def result_index(self) -> Index:
             levels=levels, codes=codes, verify_integrity=False, names=self.names
         )
         if not self.compressed and len(self.groupings) == 1:
+            # breakpoint()
             return result.get_level_values(0)
         return result
 
@@ -599,6 +610,7 @@ def _aggregate(
         is_datetimelike: bool,
         min_count: int = -1,
     ):
+        # breakpoint()
         if agg_func is libgroupby.group_nth:
             # different signature from the others
             # TODO: should we be using min_count instead of hard-coding it?
@@ -831,7 +843,7 @@ def reconstructed_codes(self) -> List[np.ndarray]:
 
     @cache_readonly
     def result_index(self):
-        breakpoint()
+        # breakpoint()
         if len(self.binlabels) != 0 and isna(self.binlabels[0]):
             return self.binlabels[1:]
 

pandas/tests/groupby/test_groupby.py

@@ -654,7 +654,6 @@ def test_groupby_as_index_agg(df):
 
         gr = df.groupby(ts.values, as_index=True)
         right = getattr(gr, attr)().reset_index(drop=True)
-
         tm.assert_frame_equal(left, right)
 
 
@@ -1750,7 +1749,6 @@ def test_empty_dataframe_groupby():
     result = df.groupby("A").sum()
     expected = DataFrame(columns=["B", "C"], dtype=np.float64)
     expected.index.name = "A"
-
     tm.assert_frame_equal(result, expected)
 
 

pandas/tests/test_multilevel.py

@@ -994,21 +994,18 @@ def test_count(self):
         with pytest.raises(KeyError, match=msg):
             frame.count(level="x")
 
-    @pytest.mark.parametrize("op", ["mad"])
+    @pytest.mark.parametrize("op", AGG_FUNCTIONS)
     @pytest.mark.parametrize("level", [0, 1])
     @pytest.mark.parametrize("skipna", [True, False])
     @pytest.mark.parametrize("sort", [True, False])
     def test_series_group_min_max(self, op, level, skipna, sort):
         # GH 17537
         grouped = self.series.groupby(level=level, sort=sort)
         # skipna=True
-        breakpoint()
         leftside = grouped.agg(lambda x: getattr(x, op)(skipna=skipna))
-        breakpoint()
         rightside = getattr(self.series, op)(level=level, skipna=skipna)
         if sort:
             rightside = rightside.sort_index(level=level)
-        breakpoint()
         tm.assert_series_equal(leftside, rightside)
 
     @pytest.mark.parametrize("op", AGG_FUNCTIONS)
@@ -1044,7 +1041,7 @@ def aggf(x):
 
         # for good measure, groupby detail
         level_index = frame._get_axis(axis).levels[level].rename(level_name)
-        breakpoint()
+
         tm.assert_index_equal(leftside._get_axis(axis), level_index)
         tm.assert_index_equal(rightside._get_axis(axis), level_index)