xfail request reformat and fill_value changes

As requested in review feedback, change tests to use request.node.addmarker in test_*.py testcases, rather than xfailing in base class tests.

Also, prototyped an attempt to create properly-sized EAs for complex numbers so that dim2 test case passes for complex (pandas-dev#54445).

Signed-off-by: Michael Tiemann <[email protected]>

Author: MichaelTiemannOSC

pandas/core/arrays/_mixins.py

@@ -517,7 +517,7 @@ def _cast_quantile_result(self, res_values: np.ndarray) -> np.ndarray:
     # numpy-like methods
 
     @classmethod
-    def _empty(cls, shape: Shape, dtype: ExtensionDtype) -> Self:
+    def _empty(cls, shape: Shape, dtype: ExtensionDtype, fill_value: object = None) -> Self:
         """
         Analogous to np.empty(shape, dtype=dtype)
 

pandas/core/arrays/arrow/array.py

@@ -247,7 +247,7 @@ def __init__(self, values: pa.Array | pa.ChunkedArray) -> None:
         self._dtype = ArrowDtype(self._pa_array.type)
 
     @classmethod
-    def _from_sequence(cls, scalars, *, dtype: Dtype | None = None, copy: bool = False):
+    def _from_sequence(cls, scalars, *, dtype: Dtype | None = None, fill_value: object | None = None, copy: bool = False):
         """
         Construct a new ExtensionArray from a sequence of scalars.
         """

pandas/core/arrays/base.py

@@ -262,7 +262,7 @@ class ExtensionArray:
     # ------------------------------------------------------------------------
 
     @classmethod
-    def _from_sequence(cls, scalars, *, dtype: Dtype | None = None, copy: bool = False):
+    def _from_sequence(cls, scalars, *, dtype: Dtype | None = None, fill_value: object = None, copy: bool = False):
         """
         Construct a new ExtensionArray from a sequence of scalars.
 
@@ -2084,7 +2084,7 @@ def _rank(
         )
 
     @classmethod
-    def _empty(cls, shape: Shape, dtype: ExtensionDtype):
+    def _empty(cls, shape: Shape, dtype: ExtensionDtype, fill_value: object = None):
         """
         Create an ExtensionArray with the given shape and dtype.
 
@@ -2096,7 +2096,7 @@ def _empty(cls, shape: Shape, dtype: ExtensionDtype):
         # Implementer note: while ExtensionDtype.empty is the public way to
         # call this method, it is still required to implement this `_empty`
         # method as well (it is called internally in pandas)
-        obj = cls._from_sequence([], dtype=dtype)
+        obj = cls._from_sequence([], dtype=dtype, fill_value=fill_value)
 
         taker = np.broadcast_to(np.intp(-1), shape)
         result = obj.take(taker, allow_fill=True)

pandas/core/arrays/categorical.py

@@ -505,7 +505,7 @@ def _internal_fill_value(self) -> int:
 
     @classmethod
     def _from_sequence(
-        cls, scalars, *, dtype: Dtype | None = None, copy: bool = False
+            cls, scalars, *, dtype: Dtype | None = None, fill_value: object = None, copy: bool = False
     ) -> Self:
         return cls(scalars, dtype=dtype, copy=copy)
 
@@ -1800,7 +1800,7 @@ def value_counts(self, dropna: bool = True) -> Series:
     # "ExtensionDtype"
     @classmethod
     def _empty(  # type: ignore[override]
-        cls, shape: Shape, dtype: CategoricalDtype
+            cls, shape: Shape, dtype: CategoricalDtype, fill_value: object = None
     ) -> Self:
         """
         Analogous to np.empty(shape, dtype=dtype)

pandas/core/arrays/datetimes.py

@@ -293,7 +293,7 @@ def _simple_new(  # type: ignore[override]
         return result
 
     @classmethod
-    def _from_sequence(cls, scalars, *, dtype=None, copy: bool = False):
+    def _from_sequence(cls, scalars, *, dtype=None, fill_value: object = None, copy: bool = False):
         return cls._from_sequence_not_strict(scalars, dtype=dtype, copy=copy)
 
     @classmethod

pandas/core/arrays/interval.py

@@ -382,6 +382,7 @@ def _from_sequence(
         scalars,
         *,
         dtype: Dtype | None = None,
+        fill_value: object = None,
         copy: bool = False,
     ) -> Self:
         return cls(scalars, dtype=dtype, copy=copy)

pandas/core/arrays/masked.py

@@ -145,13 +145,13 @@ def __init__(
         self._mask = mask
 
     @classmethod
-    def _from_sequence(cls, scalars, *, dtype=None, copy: bool = False) -> Self:
+    def _from_sequence(cls, scalars, *, dtype=None, fill_value: object = None, copy: bool = False) -> Self:
         values, mask = cls._coerce_to_array(scalars, dtype=dtype, copy=copy)
         return cls(values, mask)
 
     @classmethod
     @doc(ExtensionArray._empty)
-    def _empty(cls, shape: Shape, dtype: ExtensionDtype):
+    def _empty(cls, shape: Shape, dtype: ExtensionDtype, fill_value: object = None):
         values = np.empty(shape, dtype=dtype.type)
         values.fill(cls._internal_fill_value)
         mask = np.ones(shape, dtype=bool)

pandas/core/arrays/numpy_.py

@@ -117,7 +117,7 @@ def __init__(
 
     @classmethod
     def _from_sequence(
-        cls, scalars, *, dtype: Dtype | None = None, copy: bool = False
+        cls, scalars, *, dtype: Dtype | None = None, fill_value: object = None, copy: bool = False
     ) -> NumpyExtensionArray:
         if isinstance(dtype, NumpyEADtype):
             dtype = dtype._dtype

pandas/core/arrays/period.py

@@ -276,6 +276,7 @@ def _from_sequence(
         scalars,
         *,
         dtype: Dtype | None = None,
+        fill_value: object | None = None,
         copy: bool = False,
     ) -> Self:
         if dtype is not None:

pandas/core/arrays/sparse/array.py

@@ -583,7 +583,7 @@ def __setitem__(self, key, value) -> None:
         raise TypeError(msg)
 
     @classmethod
-    def _from_sequence(cls, scalars, *, dtype: Dtype | None = None, copy: bool = False):
+    def _from_sequence(cls, scalars, *, dtype: Dtype | None = None, fill_value: object | None = None, copy: bool = False):
         return cls(scalars, dtype=dtype)
 
     @classmethod

pandas/core/arrays/string_.py

@@ -340,7 +340,7 @@ def _validate(self):
             lib.convert_nans_to_NA(self._ndarray)
 
     @classmethod
-    def _from_sequence(cls, scalars, *, dtype: Dtype | None = None, copy: bool = False):
+    def _from_sequence(cls, scalars, *, dtype: Dtype | None = None, fill_value: object | None = None, copy: bool = False):
         if dtype and not (isinstance(dtype, str) and dtype == "string"):
             dtype = pandas_dtype(dtype)
             assert isinstance(dtype, StringDtype) and dtype.storage == "python"
@@ -377,7 +377,7 @@ def _from_sequence_of_strings(
         return cls._from_sequence(strings, dtype=dtype, copy=copy)
 
     @classmethod
-    def _empty(cls, shape, dtype) -> StringArray:
+    def _empty(cls, shape, dtype, fill_value = None) -> StringArray:
         values = np.empty(shape, dtype=object)
         values[:] = libmissing.NA
         return cls(values).astype(dtype, copy=False)

pandas/core/arrays/string_arrow.py

@@ -137,7 +137,7 @@ def __len__(self) -> int:
         return len(self._pa_array)
 
     @classmethod
-    def _from_sequence(cls, scalars, dtype: Dtype | None = None, copy: bool = False):
+    def _from_sequence(cls, scalars, dtype: Dtype | None = None, fill_value: object | None = None, copy: bool = False):
         from pandas.core.arrays.masked import BaseMaskedArray
 
         _chk_pyarrow_available()

pandas/core/arrays/timedeltas.py

@@ -229,7 +229,7 @@ def _simple_new(  # type: ignore[override]
         return result
 
     @classmethod
-    def _from_sequence(cls, data, *, dtype=None, copy: bool = False) -> Self:
+    def _from_sequence(cls, data, *, dtype=None, fill_value: object = None, copy: bool = False) -> Self:
         if dtype:
             dtype = _validate_td64_dtype(dtype)
 

pandas/core/internals/managers.py

@@ -945,8 +945,13 @@ def fast_xs(self, loc: int) -> SingleBlockManager:
         immutable_ea = isinstance(dtype, ExtensionDtype) and dtype._is_immutable
 
         if isinstance(dtype, ExtensionDtype) and not immutable_ea:
+            result = self.blocks[0].iget((slice(None), loc))
+            if isinstance(result, np.ndarray):
+                empty_fill_value = result[0]
+            else:
+                empty_fill_value = result._values_for_factorize()[0][0]
             cls = dtype.construct_array_type()
-            result = cls._empty((n,), dtype=dtype)
+            result = cls._empty((n,), dtype=dtype, fill_value=empty_fill_value)
         else:
             # error: Argument "dtype" to "empty" has incompatible type
             # "Union[Type[object], dtype[Any], ExtensionDtype, None]"; expected

pandas/tests/extension/base/dim2.py

@@ -9,6 +9,7 @@
 from pandas.core.dtypes.common import (
     is_bool_dtype,
     is_integer_dtype,
+    is_complex_dtype,
 )
 
 import pandas as pd
@@ -213,12 +214,6 @@ def test_reductions_2d_axis0(self, data, method, min_count):
 
         kwargs = {}
         if method in ["std", "var"]:
-            if data.dtype.kind == "c":
-                pytest.skip(
-                    f"{data.dtype.name} extension array"
-                    "collapses to real-valued numbers"
-                    f"with operator {method}"
-                )
             # pass ddof=0 so we get all-zero std instead of all-NA std
             kwargs["ddof"] = 0
         elif method in ["prod", "sum"]:
@@ -268,6 +263,9 @@ def get_reduction_result_dtype(dtype):
                 data = data.astype("Float64")
             if method == "mean":
                 tm.assert_extension_array_equal(result, data)
+            elif is_complex_dtype(data) and method in ["std", "var"]:
+                # std and var produce real-only results
+                tm.assert_extension_array_equal(result, data - data, check_dtype=False)
             else:
                 tm.assert_extension_array_equal(result, data - data)
 

pandas/tests/extension/base/io.py

@@ -10,9 +10,13 @@
 
 class BaseParsingTests(BaseExtensionTests):
     @pytest.mark.parametrize("engine", ["c", "python"])
-    def test_EA_types(self, engine, data):
-        if data.dtype.kind == "c":
-            pytest.skip(f"the dtype {data.dtype.name} is not supported for parsing")
+    def test_EA_types(self, engine, data, request):
+        if engine == "c" and data.dtype.kind == "c":
+            request.node.add_marker(
+                pytest.mark.xfail(
+                    reason=f"engine '{engine}' cannot parse the dtype {data.dtype.name}"
+                )
+            )
         df = pd.DataFrame({"with_dtype": pd.Series(data, dtype=str(data.dtype))})
         csv_output = df.to_csv(index=False, na_rep=np.nan)
         result = pd.read_csv(

pandas/tests/extension/base/missing.py

@@ -87,12 +87,7 @@ def test_fillna_limit_backfill(self, data_missing):
         expected = pd.Series(data_missing.take([1, 0, 1, 1, 1]))
         tm.assert_series_equal(result, expected)
 
-    def test_fillna_no_op_returns_copy(self, data):
-        if data.dtype.kind == "c":
-            pytest.skip(
-                f"no cython implementation of backfill(ndarray[{data.dtype.name}_t],"
-                f"ndarray[{data.dtype.name}_t], int64_t) in libs/algos.pxd"
-            )
+    def test_fillna_no_op_returns_copy(self, data, request):
         data = data[~data.isna()]
 
         valid = data[0]

pandas/tests/extension/base/ops.py

@@ -18,7 +18,7 @@ class BaseOpsUtil(BaseExtensionTests):
     divmod_exc: type[Exception] | None = TypeError
 
     def _get_expected_exception(
-        self, op_name: str, obj, other
+        self, op_name: str, obj, other, request
     ) -> type[Exception] | None:
         # Find the Exception, if any we expect to raise calling
         #  obj.__op_name__(other)
@@ -53,8 +53,8 @@ def get_op_from_name(self, op_name: str):
     #  case that still requires overriding _check_op or _combine, please let
     #  us know at github.com/pandas-dev/pandas/issues
     @final
-    def check_opname(self, ser: pd.Series, op_name: str, other):
-        exc = self._get_expected_exception(op_name, ser, other)
+    def check_opname(self, ser: pd.Series, op_name: str, other, request):
+        exc = self._get_expected_exception(op_name, ser, other, request)
         op = self.get_op_from_name(op_name)
 
         self._check_op(ser, op, other, op_name, exc)
@@ -90,12 +90,12 @@ def _check_op(
 
     # see comment on check_opname
     @final
-    def _check_divmod_op(self, ser: pd.Series, op, other):
+    def _check_divmod_op(self, ser: pd.Series, op, other, request):
         # check that divmod behavior matches behavior of floordiv+mod
         if op is divmod:
-            exc = self._get_expected_exception("__divmod__", ser, other)
+            exc = self._get_expected_exception("__divmod__", ser, other, request)
         else:
-            exc = self._get_expected_exception("__rdivmod__", ser, other)
+            exc = self._get_expected_exception("__rdivmod__", ser, other, request)
         if exc is None:
             result_div, result_mod = op(ser, other)
             if op is divmod:
@@ -127,47 +127,47 @@ class BaseArithmeticOpsTests(BaseOpsUtil):
     series_array_exc: type[Exception] | None = TypeError
     divmod_exc: type[Exception] | None = TypeError
 
-    def test_arith_series_with_scalar(self, data, all_arithmetic_operators):
+    def test_arith_series_with_scalar(self, data, all_arithmetic_operators, request):
         # series & scalar
         op_name = all_arithmetic_operators
         ser = pd.Series(data)
-        self.check_opname(ser, op_name, ser.iloc[0])
+        self.check_opname(ser, op_name, ser.iloc[0], request)
 
-    def test_arith_frame_with_scalar(self, data, all_arithmetic_operators):
+    def test_arith_frame_with_scalar(self, data, all_arithmetic_operators, request):
         # frame & scalar
         op_name = all_arithmetic_operators
         df = pd.DataFrame({"A": data})
-        self.check_opname(df, op_name, data[0])
+        self.check_opname(df, op_name, data[0], request)
 
-    def test_arith_series_with_array(self, data, all_arithmetic_operators):
+    def test_arith_series_with_array(self, data, all_arithmetic_operators, request):
         # ndarray & other series
         op_name = all_arithmetic_operators
         ser = pd.Series(data)
-        self.check_opname(ser, op_name, pd.Series([ser.iloc[0]] * len(ser)))
+        self.check_opname(ser, op_name, pd.Series([ser.iloc[0]] * len(ser)), request)
 
-    def test_divmod(self, data):
+    def test_divmod(self, data, request):
         ser = pd.Series(data)
-        self._check_divmod_op(ser, divmod, 1)
-        self._check_divmod_op(1, ops.rdivmod, ser)
+        self._check_divmod_op(ser, divmod, 1, request)
+        self._check_divmod_op(1, ops.rdivmod, ser, request)
 
-    def test_divmod_series_array(self, data, data_for_twos):
+    def test_divmod_series_array(self, data, data_for_twos, request):
         ser = pd.Series(data)
-        self._check_divmod_op(ser, divmod, data)
+        self._check_divmod_op(ser, divmod, data, request)
 
         other = data_for_twos
-        self._check_divmod_op(other, ops.rdivmod, ser)
+        self._check_divmod_op(other, ops.rdivmod, ser, request)
 
         other = pd.Series(other)
-        self._check_divmod_op(other, ops.rdivmod, ser)
+        self._check_divmod_op(other, ops.rdivmod, ser, request)
 
-    def test_add_series_with_extension_array(self, data):
+    def test_add_series_with_extension_array(self, data, request):
         # Check adding an ExtensionArray to a Series of the same dtype matches
         # the behavior of adding the arrays directly and then wrapping in a
         # Series.
 
         ser = pd.Series(data)
 
-        exc = self._get_expected_exception("__add__", ser, data)
+        exc = self._get_expected_exception("__add__", ser, data, request)
         if exc is not None:
             with pytest.raises(exc):
                 ser + data

pandas/tests/extension/decimal/array.py

@@ -97,7 +97,7 @@ def dtype(self):
         return self._dtype
 
     @classmethod
-    def _from_sequence(cls, scalars, dtype=None, copy=False):
+    def _from_sequence(cls, scalars, dtype=None, fill_value=None, copy=False):
         return cls(scalars)
 
     @classmethod

pandas/tests/extension/decimal/test_decimal.py

@@ -120,12 +120,12 @@ def test_fillna_limit_backfill(self, data_missing):
         ):
             super().test_fillna_limit_backfill(data_missing)
 
-    def test_fillna_no_op_returns_copy(self, data):
+    def test_fillna_no_op_returns_copy(self, data, request):
         msg = "ExtensionArray.fillna 'method' keyword is deprecated"
         with tm.assert_produces_warning(
             FutureWarning, match=msg, check_stacklevel=False
         ):
-            super().test_fillna_no_op_returns_copy(data)
+            super().test_fillna_no_op_returns_copy(data, request)
 
     def test_fillna_series(self, data_missing):
         msg = "ExtensionArray.fillna added a 'copy' keyword"
@@ -311,11 +311,11 @@ class TestArithmeticOps(base.BaseArithmeticOpsTests):
     series_array_exc = None
 
     def _get_expected_exception(
-        self, op_name: str, obj, other
+        self, op_name: str, obj, other, request
     ) -> type[Exception] | None:
         return None
 
-    def test_arith_series_with_array(self, data, all_arithmetic_operators):
+    def test_arith_series_with_array(self, data, all_arithmetic_operators, request):
         op_name = all_arithmetic_operators
         s = pd.Series(data)
 
@@ -327,13 +327,13 @@ def test_arith_series_with_array(self, data, all_arithmetic_operators):
 
         # Decimal supports ops with int, but not float
         other = pd.Series([int(d * 100) for d in data])
-        self.check_opname(s, op_name, other)
+        self.check_opname(s, op_name, other, request)
 
         if "mod" not in op_name:
-            self.check_opname(s, op_name, s * 2)
+            self.check_opname(s, op_name, s * 2, request)
 
-        self.check_opname(s, op_name, 0)
-        self.check_opname(s, op_name, 5)
+        self.check_opname(s, op_name, 0, request)
+        self.check_opname(s, op_name, 5, request)
         context.traps[decimal.DivisionByZero] = divbyzerotrap
         context.traps[decimal.InvalidOperation] = invalidoptrap
 

pandas/tests/extension/json/array.py

@@ -83,7 +83,7 @@ def __init__(self, values, dtype=None, copy=False) -> None:
         # self._values = self.values = self.data
 
     @classmethod
-    def _from_sequence(cls, scalars, dtype=None, copy=False):
+    def _from_sequence(cls, scalars, dtype=None, fill_value=None, copy=False):
         return cls(scalars)
 
     @classmethod