DEPR: fillna downcasting from object dtype (#54261)

* DEPR: fillna downcasting from object dtype

* GH ref

* suppress warning

* update test

* Update doc/source/whatsnew/v2.1.0.rst

Co-authored-by: Matthew Roeschke <[email protected]>

---------

Co-authored-by: Matthew Roeschke <[email protected]>

Author: jbrockmendel

doc/source/whatsnew/v2.2.0.rst

@@ -198,6 +198,7 @@ Deprecations
 - Deprecated strings ``T``, ``S``, ``L``, ``U``, and ``N`` denoting frequencies in :class:`Minute`, :class:`Second`, :class:`Milli`, :class:`Micro`, :class:`Nano` (:issue:`52536`)
 - Deprecated strings ``T``, ``S``, ``L``, ``U``, and ``N`` denoting units in :class:`Timedelta` (:issue:`52536`)
 - Deprecated the extension test classes ``BaseNoReduceTests``, ``BaseBooleanReduceTests``, and ``BaseNumericReduceTests``, use ``BaseReduceTests`` instead (:issue:`54663`)
+- Deprecating downcasting the results of :meth:`DataFrame.fillna`, :meth:`Series.fillna`, :meth:`DataFrame.ffill`, :meth:`Series.ffill`, :meth:`DataFrame.bfill`, :meth:`Series.bfill` in object-dtype cases. To opt in to the future version, use ``pd.set_option("future.no_silent_downcasting", True)`` (:issue:`54261`)
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_220.performance:

pandas/core/generic.py

@@ -10393,7 +10393,14 @@ def _where(
 
         # make sure we are boolean
         fill_value = bool(inplace)
-        cond = cond.fillna(fill_value)
+        with warnings.catch_warnings():
+            warnings.filterwarnings(
+                "ignore",
+                "Downcasting object dtype arrays",
+                category=FutureWarning,
+            )
+            cond = cond.fillna(fill_value)
+        cond = cond.infer_objects(copy=False)
 
         msg = "Boolean array expected for the condition, not {dtype}"
 

pandas/core/internals/blocks.py

@@ -498,7 +498,11 @@ def coerce_to_target_dtype(self, other, warn_on_upcast: bool = False) -> Block:
 
     @final
     def _maybe_downcast(
-        self, blocks: list[Block], downcast, using_cow: bool, caller: str
+        self,
+        blocks: list[Block],
+        downcast,
+        using_cow: bool,
+        caller: str,
     ) -> list[Block]:
         if downcast is False:
             return blocks
@@ -510,9 +514,29 @@ def _maybe_downcast(
             #  but ATM it breaks too much existing code.
             # split and convert the blocks
 
+            if caller == "fillna" and get_option("future.no_silent_downcasting"):
+                return blocks
+
             nbs = extend_blocks(
                 [blk.convert(using_cow=using_cow, copy=not using_cow) for blk in blocks]
             )
+            if caller == "fillna":
+                if len(nbs) != len(blocks) or not all(
+                    x.dtype == y.dtype for x, y in zip(nbs, blocks)
+                ):
+                    # GH#54261
+                    warnings.warn(
+                        "Downcasting object dtype arrays on .fillna, .ffill, .bfill "
+                        "is deprecated and will change in a future version. "
+                        "Call result.infer_objects(copy=False) instead. "
+                        "To opt-in to the future "
+                        "behavior, set "
+                        "`pd.set_option('future.no_silent_downcasting', True)`",
+                        FutureWarning,
+                        stacklevel=find_stack_level(),
+                    )
+
+            return nbs
 
         elif downcast is None:
             return blocks
@@ -1549,7 +1573,7 @@ def pad_or_backfill(
         data = extract_array(new_values, extract_numpy=True)
 
         nb = self.make_block_same_class(data, refs=refs)
-        return nb._maybe_downcast([nb], downcast, using_cow, caller="pad_or_backfill")
+        return nb._maybe_downcast([nb], downcast, using_cow, caller="fillna")
 
     @final
     def interpolate(

pandas/io/formats/xml.py

@@ -9,6 +9,7 @@
     TYPE_CHECKING,
     Any,
 )
+import warnings
 
 from pandas.errors import AbstractMethodError
 from pandas.util._decorators import doc
@@ -202,7 +203,13 @@ def process_dataframe(self) -> dict[int | str, dict[str, Any]]:
             df = df.reset_index()
 
         if self.na_rep is not None:
-            df = df.fillna(self.na_rep)
+            with warnings.catch_warnings():
+                warnings.filterwarnings(
+                    "ignore",
+                    "Downcasting object dtype arrays",
+                    category=FutureWarning,
+                )
+                df = df.fillna(self.na_rep)
 
         return df.to_dict(orient="index")
 

pandas/io/json/_json.py

@@ -1217,7 +1217,16 @@ def _try_convert_data(
             if not self.dtype:
                 if all(notna(data)):
                     return data, False
-                return data.fillna(np.nan), True
+
+                with warnings.catch_warnings():
+                    warnings.filterwarnings(
+                        "ignore",
+                        "Downcasting object dtype arrays",
+                        category=FutureWarning,
+                    )
+                    filled = data.fillna(np.nan)
+
+                return filled, True
 
             elif self.dtype is True:
                 pass

pandas/io/stata.py

@@ -2983,7 +2983,14 @@ def _prepare_data(self) -> np.rec.recarray:
         for i, col in enumerate(data):
             typ = typlist[i]
             if typ <= self._max_string_length:
-                data[col] = data[col].fillna("").apply(_pad_bytes, args=(typ,))
+                with warnings.catch_warnings():
+                    warnings.filterwarnings(
+                        "ignore",
+                        "Downcasting object dtype arrays",
+                        category=FutureWarning,
+                    )
+                    dc = data[col].fillna("")
+                data[col] = dc.apply(_pad_bytes, args=(typ,))
                 stype = f"S{typ}"
                 dtypes[col] = stype
                 data[col] = data[col].astype(stype)

pandas/plotting/_matplotlib/core.py

@@ -1538,7 +1538,13 @@ def _kind(self) -> Literal["area"]:
 
     def __init__(self, data, **kwargs) -> None:
         kwargs.setdefault("stacked", True)
-        data = data.fillna(value=0)
+        with warnings.catch_warnings():
+            warnings.filterwarnings(
+                "ignore",
+                "Downcasting object dtype arrays",
+                category=FutureWarning,
+            )
+            data = data.fillna(value=0)
         LinePlot.__init__(self, data, **kwargs)
 
         if not self.stacked:

pandas/tests/extension/test_masked.py

@@ -13,6 +13,8 @@
 be added to the array-specific tests in `pandas/tests/arrays/`.
 
 """
+import warnings
+
 import numpy as np
 import pytest
 
@@ -186,7 +188,14 @@ def _cast_pointwise_result(self, op_name: str, obj, other, pointwise_result):
 
         if sdtype.kind in "iu":
             if op_name in ("__rtruediv__", "__truediv__", "__div__"):
-                expected = expected.fillna(np.nan).astype("Float64")
+                with warnings.catch_warnings():
+                    warnings.filterwarnings(
+                        "ignore",
+                        "Downcasting object dtype arrays",
+                        category=FutureWarning,
+                    )
+                    filled = expected.fillna(np.nan)
+                expected = filled.astype("Float64")
             else:
                 # combine method result in 'biggest' (int64) dtype
                 expected = expected.astype(sdtype)

pandas/tests/frame/indexing/test_where.py

@@ -96,6 +96,7 @@ def test_where_upcasting(self):
 
         tm.assert_series_equal(result, expected)
 
+    @pytest.mark.filterwarnings("ignore:Downcasting object dtype arrays:FutureWarning")
     def test_where_alignment(self, where_frame, float_string_frame):
         # aligning
         def _check_align(df, cond, other, check_dtypes=True):
@@ -170,6 +171,7 @@ def test_where_invalid(self):
         with pytest.raises(ValueError, match=msg):
             df.mask(0)
 
+    @pytest.mark.filterwarnings("ignore:Downcasting object dtype arrays:FutureWarning")
     def test_where_set(self, where_frame, float_string_frame, mixed_int_frame):
         # where inplace
 

pandas/tests/frame/methods/test_fillna.py

@@ -360,7 +360,9 @@ def test_fillna_dtype_conversion(self):
         expected = Series([np.dtype("object")] * 5, index=[1, 2, 3, 4, 5])
         tm.assert_series_equal(result, expected)
 
-        result = df.fillna(1)
+        msg = "Downcasting object dtype arrays"
+        with tm.assert_produces_warning(FutureWarning, match=msg):
+            result = df.fillna(1)
         expected = DataFrame(1, index=["A", "B", "C"], columns=[1, 2, 3, 4, 5])
         tm.assert_frame_equal(result, expected)
 
@@ -817,7 +819,8 @@ def test_fillna_nones_inplace():
         [[None, None], [None, None]],
         columns=["A", "B"],
     )
-    with tm.assert_produces_warning(False):
+    msg = "Downcasting object dtype arrays"
+    with tm.assert_produces_warning(FutureWarning, match=msg):
         df.fillna(value={"A": 1, "B": 2}, inplace=True)
 
     expected = DataFrame([[1, 2], [1, 2]], columns=["A", "B"])