BUG: DataFrame.groupby with as_index=False shouldn't modify grouping columns

Author: rhshadrach

asv_bench/benchmarks/arithmetic.py

@@ -101,6 +101,59 @@ def time_frame_op_with_series_axis1(self, opname):
         getattr(operator, opname)(self.df, self.ser)
 
 
+class FrameWithFrameWide:
+    # Many-columns, mixed dtypes
+
+    params = [
+        [
+            # GH#32779 has discussion of which operators are included here
+            operator.add,
+            operator.floordiv,
+            operator.gt,
+        ]
+    ]
+    param_names = ["op"]
+
+    def setup(self, op):
+        # we choose dtypes so as to make the blocks
+        #  a) not perfectly match between right and left
+        #  b) appreciably bigger than single columns
+        n_cols = 2000
+        n_rows = 500
+
+        # construct dataframe with 2 blocks
+        arr1 = np.random.randn(n_rows, int(n_cols / 2)).astype("f8")
+        arr2 = np.random.randn(n_rows, int(n_cols / 2)).astype("f4")
+        df = pd.concat(
+            [pd.DataFrame(arr1), pd.DataFrame(arr2)], axis=1, ignore_index=True,
+        )
+        # should already be the case, but just to be sure
+        df._consolidate_inplace()
+
+        # TODO: GH#33198 the setting here shoudlnt need two steps
+        arr1 = np.random.randn(n_rows, int(n_cols / 4)).astype("f8")
+        arr2 = np.random.randn(n_rows, int(n_cols / 2)).astype("i8")
+        arr3 = np.random.randn(n_rows, int(n_cols / 4)).astype("f8")
+        df2 = pd.concat(
+            [pd.DataFrame(arr1), pd.DataFrame(arr2), pd.DataFrame(arr3)],
+            axis=1,
+            ignore_index=True,
+        )
+        # should already be the case, but just to be sure
+        df2._consolidate_inplace()
+
+        self.left = df
+        self.right = df2
+
+    def time_op_different_blocks(self, op):
+        # blocks (and dtypes) are not aligned
+        op(self.left, self.right)
+
+    def time_op_same_blocks(self, op):
+        # blocks (and dtypes) are aligned
+        op(self.left, self.left)
+
+
 class Ops:
 
     params = [[True, False], ["default", 1]]

doc/source/whatsnew/v1.1.0.rst

@@ -561,6 +561,63 @@ Assignment to multiple columns of a :class:`DataFrame` when some of the columns
    df[['a', 'c']] = 1
    df
 
+.. _whatsnew_110.api_breaking.groupby_nunique:
+
+Using groupby with ``nunique`` and ``as_index=True``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Using :meth:`DataFrame.groupby` with ``as_index=True`` and the aggregation ``nunique`` would include the grouping column(s) in the columns of the result. Now, the grouping columns only appear in the index. This is now consistent with other aggregation functions. (:issue:`32579`)
+
+.. ipython:: python
+
+   df = pd.DataFrame({"a": ["x", "x", "y", "y"], "b": [1, 1, 2, 3]})
+   df
+
+*Previous behavior*:
+
+.. code-block:: ipython
+
+   In [3]: df.groupby("a", as_index=True).nunique()
+   Out[4]:
+      a  b
+   a
+   x  1  1
+   y  1  2
+
+*New behavior*:
+
+.. ipython:: python
+
+   df.groupby("a", as_index=True).nunique()
+
+.. _whatsnew_110.api_breaking.groupby_as_index_false:
+
+Using groupby with ``as_index=False``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Using :meth:`DataFrame.groupby` with ``as_index=False`` and the function ``idxmax``, ``idxmin``, ``mad``, ``nunique``, or ``skew`` would modify the grouping column. Now, the grouping column remains unchanged. (:issue:`21090`)
+
+.. ipython:: python
+
+   df = pd.DataFrame({"a": ["x", "x", "y", "y"], "b": [1, 1, 2, 3]})
+   df
+
+*Previous behavior*:
+
+.. code-block:: ipython
+
+   In [3]: df.groupby("a", as_index=False).nunique()
+   Out[4]:
+      a  b
+   0  1  1
+   1  1  2
+
+*New behavior*:
+
+.. ipython:: python
+
+   df.groupby("a", as_index=False).nunique()
+
 .. _whatsnew_110.deprecations:
 
 Deprecations
@@ -611,7 +668,7 @@ Performance improvements
   and :meth:`~pandas.core.groupby.groupby.Groupby.last` (:issue:`34178`)
 - Performance improvement in :func:`factorize` for nullable (integer and boolean) dtypes (:issue:`33064`).
 - Performance improvement in reductions (sum, prod, min, max) for nullable (integer and boolean) dtypes (:issue:`30982`, :issue:`33261`, :issue:`33442`).
-
+- Performance improvement in arithmetic operations between two :class:`DataFrame` objects (:issue:`32779`)
 
 .. ---------------------------------------------------------------------------
 
@@ -816,9 +873,10 @@ Groupby/resample/rolling
 - Bug in :meth:`DataFrame.groupby` where a ``ValueError`` would be raised when grouping by a categorical column with read-only categories and ``sort=False`` (:issue:`33410`)
 - Bug in :meth:`GroupBy.first` and :meth:`GroupBy.last` where None is not preserved in object dtype (:issue:`32800`)
 - Bug in :meth:`Rolling.min` and :meth:`Rolling.max`: Growing memory usage after multiple calls when using a fixed window (:issue:`30726`)
+- Bug in :meth:`Series.groupby` would raise ``ValueError`` when grouping by :class:`PeriodIndex` level (:issue:`34010`)
 - Bug in :meth:`GroupBy.agg`, :meth:`GroupBy.transform`, and :meth:`GroupBy.resample` where subclasses are not preserved (:issue:`28330`)
 - Bug in :meth:`GroupBy.rolling.apply` ignores args and kwargs parameters (:issue:`33433`)
-
+- Bug in :meth:`DataFrameGroupby.std` and :meth:`DataFrameGroupby.sem` would modify grouped-by columns when ``as_index=False`` (:issue:`10355`)
 
 Reshaping
 ^^^^^^^^^
@@ -883,6 +941,7 @@ Other
 - Bug in :meth:`DataFrame.__dir__` caused a segfault when using unicode surrogates in a column name (:issue:`25509`)
 - Bug in :meth:`DataFrame.plot.scatter` caused an error when plotting variable marker sizes (:issue:`32904`)
 - :class:`IntegerArray` now implements the ``sum`` operation (:issue:`33172`)
+- More informative error message with ``np.min`` or ``np.max`` on unordered :class:`Categorical` (:issue:`33115`)
 - Bug in :class:`Tick` comparisons raising ``TypeError`` when comparing against timedelta-like objects (:issue:`34088`)
 
 .. ---------------------------------------------------------------------------

pandas/_libs/index.pyx

@@ -22,7 +22,8 @@ cnp.import_array()
 from pandas._libs cimport util
 
 from pandas._libs.tslibs.nattype cimport c_NaT as NaT
-from pandas._libs.tslibs.base cimport ABCTimestamp, ABCTimedelta, ABCPeriod
+from pandas._libs.tslibs.base cimport ABCTimestamp, ABCTimedelta
+from pandas._libs.tslibs.period cimport is_period_object
 
 from pandas._libs.hashtable cimport HashTable
 
@@ -479,7 +480,7 @@ cdef class PeriodEngine(Int64Engine):
     cdef int64_t _unbox_scalar(self, scalar) except? -1:
         if scalar is NaT:
             return scalar.value
-        if isinstance(scalar, ABCPeriod):
+        if is_period_object(scalar):
             # NB: we assume that we have the correct freq here.
             return scalar.ordinal
         raise TypeError(scalar)

pandas/_libs/internals.pyx

@@ -49,7 +49,7 @@ cdef class BlockPlacement:
         else:
             # Cython memoryview interface requires ndarray to be writeable.
             arr = np.require(val, dtype=np.int64, requirements='W')
-            assert arr.ndim == 1
+            assert arr.ndim == 1, arr.shape
             self._as_array = arr
             self._has_array = True
 

pandas/_libs/lib.pyx

@@ -74,7 +74,7 @@ from pandas._libs.tslibs.nattype cimport (
 from pandas._libs.tslibs.conversion cimport convert_to_tsobject
 from pandas._libs.tslibs.timedeltas cimport convert_to_timedelta64
 from pandas._libs.tslibs.timezones cimport get_timezone, tz_compare
-from pandas._libs.tslibs.base cimport is_period_object
+from pandas._libs.tslibs.period cimport is_period_object
 
 from pandas._libs.missing cimport (
     checknull,

pandas/_libs/tslibs/base.pxd

@@ -6,10 +6,3 @@ cdef class ABCTimedelta(timedelta):
 
 cdef class ABCTimestamp(datetime):
     pass
-
-
-cdef class ABCPeriod:
-    pass
-
-
-cdef bint is_period_object(object obj)

pandas/_libs/tslibs/base.pyx

@@ -14,11 +14,3 @@ cdef class ABCTimedelta(timedelta):
 
 cdef class ABCTimestamp(datetime):
     pass
-
-
-cdef class ABCPeriod:
-    pass
-
-
-cdef bint is_period_object(object obj):
-    return isinstance(obj, ABCPeriod)

pandas/_libs/tslibs/conversion.pyx

@@ -13,7 +13,7 @@ from cpython.datetime cimport (datetime, time, tzinfo,
                                PyDateTime_IMPORT)
 PyDateTime_IMPORT
 
-from pandas._libs.tslibs.base cimport ABCTimestamp, is_period_object
+from pandas._libs.tslibs.base cimport ABCTimestamp
 
 from pandas._libs.tslibs.np_datetime cimport (
     check_dts_bounds, npy_datetimestruct, pandas_datetime_to_datetimestruct,
@@ -290,10 +290,11 @@ cdef convert_to_tsobject(object ts, object tz, object unit,
         # Keep the converter same as PyDateTime's
         ts = datetime.combine(ts, time())
         return convert_datetime_to_tsobject(ts, tz)
-    elif is_period_object(ts):
-        raise ValueError("Cannot convert Period to Timestamp "
-                         "unambiguously. Use to_timestamp")
     else:
+        from .period import Period
+        if isinstance(ts, Period):
+            raise ValueError("Cannot convert Period to Timestamp "
+                             "unambiguously. Use to_timestamp")
         raise TypeError(f'Cannot convert input [{ts}] of type {type(ts)} to '
                         f'Timestamp')
 

pandas/_libs/tslibs/nattype.pyx

@@ -29,7 +29,6 @@ from pandas._libs.tslibs.np_datetime cimport (
     get_timedelta64_value,
 )
 cimport pandas._libs.tslibs.util as util
-from pandas._libs.tslibs.base cimport is_period_object
 
 
 # ----------------------------------------------------------------------
@@ -149,7 +148,7 @@ cdef class _NaT(datetime):
         elif util.is_offset_object(other):
             return c_NaT
 
-        elif util.is_integer_object(other) or is_period_object(other):
+        elif util.is_integer_object(other):
             # For Period compat
             # TODO: the integer behavior is deprecated, remove it
             return c_NaT
@@ -163,6 +162,7 @@ cdef class _NaT(datetime):
                 return result
             raise TypeError(f"Cannot add NaT to ndarray with dtype {other.dtype}")
 
+        # Includes Period going through here
         return NotImplemented
 
     def __sub__(self, other):
@@ -185,7 +185,7 @@ cdef class _NaT(datetime):
         elif util.is_offset_object(other):
             return c_NaT
 
-        elif util.is_integer_object(other) or is_period_object(other):
+        elif util.is_integer_object(other):
             # For Period compat
             # TODO: the integer behavior is deprecated, remove it
             return c_NaT
@@ -216,6 +216,7 @@ cdef class _NaT(datetime):
                 f"Cannot subtract NaT from ndarray with dtype {other.dtype}"
             )
 
+        # Includes Period going through here
         return NotImplemented
 
     def __pos__(self):