BUG: setting pd.NA into Series casts to object (#45431)

Author: jbrockmendel

doc/source/whatsnew/v1.5.0.rst

@@ -214,6 +214,7 @@ Indexing
 - Bug in :meth:`DataFrame.iloc` where indexing a single row on a :class:`DataFrame` with a single ExtensionDtype column gave a copy instead of a view on the underlying data (:issue:`45241`)
 - Bug in :meth:`Series.__setitem__` with a non-integer :class:`Index` when using an integer key to set a value that cannot be set inplace where a ``ValueError`` was raised insead of casting to a common dtype (:issue:`45070`)
 - Bug when setting a value too large for a :class:`Series` dtype failing to coerce to a common type (:issue:`26049`, :issue:`32878`)
+- Bug in :meth:`Series.__setitem__` where setting :attr:`NA` into a numeric-dtpye :class:`Series` would incorrectly upcast to object-dtype rather than treating the value as ``np.nan`` (:issue:`44199`)
 -
 
 Missing

pandas/core/frame.py

@@ -8887,9 +8887,9 @@ def applymap(
         >>> df_copy = df.copy()
         >>> df_copy.iloc[0, 0] = pd.NA
         >>> df_copy.applymap(lambda x: len(str(x)), na_action='ignore')
-              0  1
-        0  <NA>  4
-        1     5  5
+             0  1
+        0  NaN  4
+        1  5.0  5
 
         Note that a vectorized version of `func` often exists, which will
         be much faster. You could square each number elementwise.

pandas/core/indexes/base.py

@@ -5233,7 +5233,8 @@ def putmask(self, mask, value) -> Index:
         if noop:
             return self.copy()
 
-        if value is None and (self._is_numeric_dtype or self.dtype == object):
+        if self.dtype != object and is_valid_na_for_dtype(value, self.dtype):
+            # e.g. None -> np.nan, see also Block._standardize_fill_value
             value = self._na_value
         try:
             converted = self._validate_fill_value(value)

pandas/core/internals/blocks.py

@@ -878,6 +878,12 @@ def _replace_coerce(
 
     # ---------------------------------------------------------------------
 
+    def _standardize_fill_value(self, value):
+        # if we are passed a scalar None, convert it here
+        if self.dtype != _dtype_obj and is_valid_na_for_dtype(value, self.dtype):
+            value = self.fill_value
+        return value
+
     def _maybe_squeeze_arg(self, arg: np.ndarray) -> np.ndarray:
         """
         For compatibility with 1D-only ExtensionArrays.
@@ -911,10 +917,7 @@ def setitem(self, indexer, value):
         be a compatible shape.
         """
 
-        # coerce None values, if appropriate
-        if value is None:
-            if self.is_numeric:
-                value = np.nan
+        value = self._standardize_fill_value(value)
 
         # coerce if block dtype can store value
         if not self._can_hold_element(value):
@@ -968,9 +971,7 @@ def putmask(self, mask, new) -> list[Block]:
         if new is lib.no_default:
             new = self.fill_value
 
-        # if we are passed a scalar None, convert it here
-        if not self.is_object and is_valid_na_for_dtype(new, self.dtype):
-            new = self.fill_value
+        new = self._standardize_fill_value(new)
 
         if self._can_hold_element(new):
             putmask_without_repeat(values.T, mask, new)
@@ -1152,8 +1153,7 @@ def shift(self, periods: int, axis: int = 0, fill_value: Any = None) -> list[Blo
             # see test_shift_object_non_scalar_fill
             raise ValueError("fill_value must be a scalar")
 
-        if is_valid_na_for_dtype(fill_value, self.dtype) and self.dtype != _dtype_obj:
-            fill_value = self.fill_value
+        fill_value = self._standardize_fill_value(fill_value)
 
         if not self._can_hold_element(fill_value):
             nb = self.coerce_to_target_dtype(fill_value)
@@ -1196,8 +1196,7 @@ def where(self, other, cond) -> list[Block]:
         if other is lib.no_default:
             other = self.fill_value
 
-        if is_valid_na_for_dtype(other, self.dtype) and self.dtype != _dtype_obj:
-            other = self.fill_value
+        other = self._standardize_fill_value(other)
 
         if not self._can_hold_element(other):
             # we cannot coerce, return a compat dtype

pandas/tests/series/indexing/test_setitem.py

@@ -9,6 +9,7 @@
 from pandas.core.dtypes.common import is_list_like
 
 from pandas import (
+    NA,
     Categorical,
     DataFrame,
     DatetimeIndex,
@@ -770,11 +771,13 @@ def test_index_putmask(self, obj, key, expected, val):
     ],
 )
 class TestSetitemCastingEquivalents(SetitemCastingEquivalents):
-    @pytest.fixture(params=[np.nan, np.float64("NaN")])
+    @pytest.fixture(params=[np.nan, np.float64("NaN"), None, NA])
     def val(self, request):
         """
-        One python float NaN, one np.float64.  Only np.float64 has a `dtype`
-        attribute.
+        NA values that should generally be valid_na for *all* dtypes.
+
+        Include both python float NaN and np.float64; only np.float64 has a
+        `dtype` attribute.
         """
         return request.param