API: honor copy=True when passing dict to DataFrame

doc/source/whatsnew/v1.3.0.rst

@@ -39,6 +39,21 @@ For example:
 ``'table'`` option that performs the windowing operation over an entire :class:`DataFrame`.
 See ref:`window.overview` for performance and functional benefits. (:issue:`15095`, :issue:`38995`)
 
+.. _whatsnew_130.dataframe_honors_copy_with_dict:
+
+DataFrame constructor honors ``copy=False`` With Dict
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+When passing a dictionary to :class:`DataFrame` with (the default) ``copy=False``,
+a copy will no longer be made (:issue:`32960`)
+
+.. ipython:: python
+
+    arr = np.array([1, 2, 3])
+    df = pd.DataFrame({"A": arr, "B": arr.copy()})
+    arr[0] = 0
+    assert df.iloc[0, 0] == 0
+
 .. _whatsnew_130.enhancements.other:
 
 Other enhancements

pandas/_testing/__init__.py

@@ -473,7 +473,7 @@ def getPeriodData(nper=None):
 # make frame
 def makeTimeDataFrame(nper=None, freq="B"):
     data = getTimeSeriesData(nper, freq)
-    return DataFrame(data)
+    return DataFrame(data)._consolidate()
 
 
 def makeDataFrame() -> DataFrame:

pandas/conftest.py

@@ -695,7 +695,7 @@ def float_frame():
 
     [30 rows x 4 columns]
     """
-    return DataFrame(tm.getSeriesData())
+    return DataFrame(tm.getSeriesData())._consolidate()
 
 
 # ----------------------------------------------------------------

pandas/core/frame.py

@@ -534,7 +534,7 @@ def __init__(
             )
 
         elif isinstance(data, dict):
-            mgr = init_dict(data, index, columns, dtype=dtype)
+            mgr = init_dict(data, index, columns, dtype=dtype, copy=copy)
         elif isinstance(data, ma.MaskedArray):
             import numpy.ma.mrecords as mrecords
 

pandas/core/groupby/groupby.py

@@ -1763,7 +1763,9 @@ def describe(self, **kwargs):
             result = self.apply(lambda x: x.describe(**kwargs))
             if self.axis == 1:
                 return result.T
-            return result.unstack()
+            # FIXME: not being consolidated breaks
+            #  test_describe_with_duplicate_output_column_names
+            return result._consolidate().unstack()
 
     @final
     def resample(self, rule, *args, **kwargs):

pandas/core/internals/construction.py

@@ -78,6 +78,7 @@ def arrays_to_mgr(
     columns,
     dtype: Optional[DtypeObj] = None,
     verify_integrity: bool = True,
+    consolidate: bool = True,
 ):
     """
     Segregate Series based on type and coerce into matrices.
@@ -104,7 +105,9 @@ def arrays_to_mgr(
     # from BlockManager perspective
     axes = [columns, index]
 
-    return create_block_manager_from_arrays(arrays, arr_names, axes)
+    return create_block_manager_from_arrays(
+        arrays, arr_names, axes, consolidate=consolidate
+    )
 
 
 def masked_rec_array_to_mgr(
@@ -153,7 +156,13 @@ def masked_rec_array_to_mgr(
 # DataFrame Constructor Interface
 
 
-def init_ndarray(values, index, columns, dtype: Optional[DtypeObj], copy: bool):
+def init_ndarray(
+    values,
+    index,
+    columns,
+    dtype: Optional[DtypeObj],
+    copy: bool,
+):
     # input must be a ndarray, list, Series, index
 
     if isinstance(values, ABCSeries):
@@ -235,7 +244,14 @@ def init_ndarray(values, index, columns, dtype: Optional[DtypeObj], copy: bool):
     return create_block_manager_from_blocks(block_values, [columns, index])
 
 
-def init_dict(data: Dict, index, columns, dtype: Optional[DtypeObj] = None):
+def init_dict(
+    data: Dict,
+    index,
+    columns,
+    *,
+    dtype: Optional[DtypeObj] = None,
+    copy: bool = True,
+):
     """
     Segregate Series based on type and coerce into matrices.
     Needs to handle a lot of exceptional cases.
@@ -269,6 +285,8 @@ def init_dict(data: Dict, index, columns, dtype: Optional[DtypeObj] = None):
             val = construct_1d_arraylike_from_scalar(np.nan, len(index), nan_dtype)
             arrays.loc[missing] = [val] * missing.sum()
 
+        arrays = list(arrays)
+
     else:
         keys = list(data.keys())
         columns = data_names = Index(keys)
@@ -279,7 +297,15 @@ def init_dict(data: Dict, index, columns, dtype: Optional[DtypeObj] = None):
         arrays = [
             arr if not is_datetime64tz_dtype(arr) else arr.copy() for arr in arrays
         ]
-    return arrays_to_mgr(arrays, data_names, index, columns, dtype=dtype)
+
+    if copy:
+        # arrays_to_mgr (via form_blocks) won't make copies for EAs
+        arrays = [x if not is_extension_array_dtype(x) else x.copy() for x in arrays]
+        # TODO: can we get rid of the dt64tz special case above?
+
+    return arrays_to_mgr(
+        arrays, data_names, index, columns, dtype=dtype, consolidate=copy
+    )
 
 
 def nested_data_to_arrays(

pandas/core/internals/managers.py

@@ -41,7 +41,7 @@
 import pandas.core.algorithms as algos
 from pandas.core.arrays.sparse import SparseDtype
 from pandas.core.base import PandasObject
-from pandas.core.construction import extract_array
+from pandas.core.construction import ensure_wrapped_if_datetimelike, extract_array
 from pandas.core.indexers import maybe_convert_indices
 from pandas.core.indexes.api import Index, ensure_index
 from pandas.core.internals.blocks import (
@@ -955,6 +955,8 @@ def fast_xs(self, loc: int) -> ArrayLike:
         else:
             result = np.empty(n, dtype=dtype)
 
+        result = ensure_wrapped_if_datetimelike(result)
+
         for blk in self.blocks:
             # Such assignment may incorrectly coerce NaT to None
             # result[blk.mgr_locs] = blk._slice((slice(None), loc))
@@ -1665,7 +1667,9 @@ def fast_xs(self, loc):
 # Constructor Helpers
 
 
-def create_block_manager_from_blocks(blocks, axes: List[Index]) -> BlockManager:
+def create_block_manager_from_blocks(
+    blocks, axes: List[Index], consolidate: bool = True
+) -> BlockManager:
     try:
         if len(blocks) == 1 and not isinstance(blocks[0], Block):
             # if blocks[0] is of length 0, return empty blocks
@@ -1682,7 +1686,8 @@ def create_block_manager_from_blocks(blocks, axes: List[Index]) -> BlockManager:
                 ]
 
         mgr = BlockManager(blocks, axes)
-        mgr._consolidate_inplace()
+        if consolidate:
+            mgr._consolidate_inplace()
         return mgr
 
     except ValueError as e:
@@ -1692,7 +1697,10 @@ def create_block_manager_from_blocks(blocks, axes: List[Index]) -> BlockManager:
 
 
 def create_block_manager_from_arrays(
-    arrays, names: Index, axes: List[Index]
+    arrays,
+    names: Index,
+    axes: List[Index],
+    consolidate: bool = True,
 ) -> BlockManager:
     assert isinstance(names, Index)
     assert isinstance(axes, list)
@@ -1702,12 +1710,13 @@ def create_block_manager_from_arrays(
     # Note: just calling extract_array breaks tests that patch PandasArray._typ.
     arrays = [x if not isinstance(x, ABCPandasArray) else x.to_numpy() for x in arrays]
     try:
-        blocks = _form_blocks(arrays, names, axes)
+        blocks = _form_blocks(arrays, names, axes, consolidate)
         mgr = BlockManager(blocks, axes)
-        mgr._consolidate_inplace()
-        return mgr
     except ValueError as e:
         raise construction_error(len(arrays), arrays[0].shape, axes, e)
+    if consolidate:
+        mgr._consolidate_inplace()
+    return mgr
 
 
 def construction_error(tot_items, block_shape, axes, e=None):
@@ -1734,7 +1743,7 @@ def construction_error(tot_items, block_shape, axes, e=None):
 # -----------------------------------------------------------------------
 
 
-def _form_blocks(arrays, names: Index, axes) -> List[Block]:
+def _form_blocks(arrays, names: Index, axes, consolidate: bool) -> List[Block]:
     # put "leftover" items in float bucket, where else?
     # generalize?
     items_dict: DefaultDict[str, List] = defaultdict(list)
@@ -1760,23 +1769,31 @@ def _form_blocks(arrays, names: Index, axes) -> List[Block]:
 
     blocks: List[Block] = []
     if len(items_dict["FloatBlock"]):
-        float_blocks = _multi_blockify(items_dict["FloatBlock"])
+        float_blocks = _multi_blockify(
+            items_dict["FloatBlock"], consolidate=consolidate
+        )
         blocks.extend(float_blocks)
 
     if len(items_dict["ComplexBlock"]):
-        complex_blocks = _multi_blockify(items_dict["ComplexBlock"])
+        complex_blocks = _multi_blockify(
+            items_dict["ComplexBlock"], consolidate=consolidate
+        )
         blocks.extend(complex_blocks)
 
     if len(items_dict["TimeDeltaBlock"]):
-        timedelta_blocks = _multi_blockify(items_dict["TimeDeltaBlock"])
+        timedelta_blocks = _multi_blockify(
+            items_dict["TimeDeltaBlock"], consolidate=consolidate
+        )
         blocks.extend(timedelta_blocks)
 
     if len(items_dict["IntBlock"]):
-        int_blocks = _multi_blockify(items_dict["IntBlock"])
+        int_blocks = _multi_blockify(items_dict["IntBlock"], consolidate=consolidate)
         blocks.extend(int_blocks)
 
     if len(items_dict["DatetimeBlock"]):
-        datetime_blocks = _simple_blockify(items_dict["DatetimeBlock"], DT64NS_DTYPE)
+        datetime_blocks = _simple_blockify(
+            items_dict["DatetimeBlock"], DT64NS_DTYPE, consolidate=consolidate
+        )
         blocks.extend(datetime_blocks)
 
     if len(items_dict["DatetimeTZBlock"]):
@@ -1787,11 +1804,15 @@ def _form_blocks(arrays, names: Index, axes) -> List[Block]:
         blocks.extend(dttz_blocks)
 
     if len(items_dict["BoolBlock"]):
-        bool_blocks = _simple_blockify(items_dict["BoolBlock"], np.bool_)
+        bool_blocks = _simple_blockify(
+            items_dict["BoolBlock"], np.bool_, consolidate=consolidate
+        )
         blocks.extend(bool_blocks)
 
     if len(items_dict["ObjectBlock"]) > 0:
-        object_blocks = _simple_blockify(items_dict["ObjectBlock"], np.object_)
+        object_blocks = _simple_blockify(
+            items_dict["ObjectBlock"], np.object_, consolidate=consolidate
+        )
         blocks.extend(object_blocks)
 
     if len(items_dict["CategoricalBlock"]) > 0:
@@ -1830,11 +1851,14 @@ def _form_blocks(arrays, names: Index, axes) -> List[Block]:
     return blocks
 
 
-def _simple_blockify(tuples, dtype) -> List[Block]:
+def _simple_blockify(tuples, dtype, consolidate: bool) -> List[Block]:
     """
     return a single array of a block that has a single dtype; if dtype is
     not None, coerce to this dtype
     """
+    if not consolidate:
+        return _tuples_to_blocks_no_consolidate(tuples, dtype=dtype)
+
     values, placement = _stack_arrays(tuples, dtype)
 
     # TODO: CHECK DTYPE?
@@ -1845,8 +1869,12 @@ def _simple_blockify(tuples, dtype) -> List[Block]:
     return [block]
 
 
-def _multi_blockify(tuples, dtype: Optional[Dtype] = None):
+def _multi_blockify(tuples, dtype: Optional[Dtype] = None, consolidate: bool = True):
     """ return an array of blocks that potentially have different dtypes """
+
+    if not consolidate:
+        return _tuples_to_blocks_no_consolidate(tuples, dtype=dtype)
+
     # group by dtype
     grouper = itertools.groupby(tuples, lambda x: x[2].dtype)
 
@@ -1861,6 +1889,18 @@ def _multi_blockify(tuples, dtype: Optional[Dtype] = None):
     return new_blocks
 
 
+def _tuples_to_blocks_no_consolidate(tuples, dtype: Optional[DtypeObj]) -> List[Block]:
+    # tuples produced within _form_blocks are of the form (placement, whatever, array)
+    if dtype is not None:
+        return [
+            make_block(
+                np.atleast_2d(x[2].astype(dtype, copy=False)), placement=x[0], ndim=2
+            )
+            for x in tuples
+        ]
+    return [make_block(np.atleast_2d(x[2]), placement=x[0], ndim=2) for x in tuples]
+
+
 def _stack_arrays(tuples, dtype):
 
     # fml

pandas/tests/arithmetic/test_numeric.py

@@ -536,9 +536,7 @@ def test_df_mod_zero_df(self):
         # GH#3590, modulo as ints
         df = pd.DataFrame({"first": [3, 4, 5, 8], "second": [0, 0, 0, 3]})
 
-        # this is technically wrong, as the integer portion is coerced to float
-        # ###
-        first = Series([0, 0, 0, 0], dtype="float64")
+        first = Series([0, 0, 0, 0], dtype="int64")
         second = Series([np.nan, np.nan, np.nan, 0])
         expected = pd.DataFrame({"first": first, "second": second})
         result = df % df

pandas/tests/frame/test_arithmetic.py

@@ -1305,7 +1305,7 @@ def test_strings_to_numbers_comparisons_raises(self, compare_operators_no_eq_ne)
             f(df, 0)
 
     def test_comparison_protected_from_errstate(self):
-        missing_df = tm.makeDataFrame()
+        missing_df = tm.makeDataFrame()._consolidate()
         missing_df.iloc[0]["A"] = np.nan
         with np.errstate(invalid="ignore"):
             expected = missing_df.values < 0