Merge branch 'master' into misc/remove-docs

Author: jameslamb

.github/workflows/ci.yml

@@ -163,10 +163,7 @@ jobs:
         pytest pandas/tests/resample/
         pytest pandas/tests/reshape/merge
         pytest pandas/tests/series/
-
-        # indexing subset (temporary since other tests don't pass yet)
-        pytest pandas/tests/indexing/multiindex/test_setitem.py::TestMultiIndexSetItem::test_astype_assignment_with_dups
-        pytest pandas/tests/indexing/multiindex/test_setitem.py::TestMultiIndexSetItem::test_frame_setitem_multi_column
+        pytest pandas/tests/indexing/
 
         pytest pandas/tests/api/
         pytest pandas/tests/apply/

pandas/_libs/hashing.pyi

@@ -0,0 +1,7 @@
+import numpy as np
+
+def hash_object_array(
+    arr: np.ndarray,  # np.ndarray[object]
+    key: str,
+    encoding: str = ...,
+) -> np.ndarray: ...  # np.ndarray[np.uint64]

pandas/_libs/index.pyx

@@ -259,11 +259,11 @@ cdef class IndexEngine:
         self.monotonic_inc = 0
         self.monotonic_dec = 0
 
-    def get_indexer(self, values):
+    def get_indexer(self, ndarray values):
         self._ensure_mapping_populated()
         return self.mapping.lookup(values)
 
-    def get_indexer_non_unique(self, targets):
+    def get_indexer_non_unique(self, ndarray targets):
         """
         Return an indexer suitable for taking from a non unique index
         return the labels in the same order as the target
@@ -451,11 +451,11 @@ cdef class DatetimeEngine(Int64Engine):
         except KeyError:
             raise KeyError(val)
 
-    def get_indexer_non_unique(self, targets):
+    def get_indexer_non_unique(self, ndarray targets):
         # we may get datetime64[ns] or timedelta64[ns], cast these to int64
         return super().get_indexer_non_unique(targets.view("i8"))
 
-    def get_indexer(self, values):
+    def get_indexer(self, ndarray values):
         self._ensure_mapping_populated()
         if values.dtype != self._get_box_dtype():
             return np.repeat(-1, len(values)).astype('i4')
@@ -594,15 +594,15 @@ cdef class BaseMultiIndexCodesEngine:
                        in zip(self.levels, zip(*target))]
         return self._codes_to_ints(np.array(level_codes, dtype='uint64').T)
 
-    def get_indexer_no_fill(self, object target) -> np.ndarray:
+    def get_indexer(self, ndarray[object] target) -> np.ndarray:
         """
         Returns an array giving the positions of each value of `target` in
         `self.values`, where -1 represents a value in `target` which does not
         appear in `self.values`
 
         Parameters
         ----------
-        target : list-like of keys
+        target : ndarray[object]
             Each key is a tuple, with a label for each level of the index
 
         Returns
@@ -613,8 +613,8 @@ cdef class BaseMultiIndexCodesEngine:
         lab_ints = self._extract_level_codes(target)
         return self._base.get_indexer(self, lab_ints)
 
-    def get_indexer(self, object target, object values = None,
-                    object method = None, object limit = None) -> np.ndarray:
+    def get_indexer_with_fill(self, ndarray target, ndarray values,
+                              str method, object limit) -> np.ndarray:
         """
         Returns an array giving the positions of each value of `target` in
         `values`, where -1 represents a value in `target` which does not
@@ -630,25 +630,22 @@ cdef class BaseMultiIndexCodesEngine:
 
         Parameters
         ----------
-        target: list-like of tuples
+        target: ndarray[object] of tuples
             need not be sorted, but all must have the same length, which must be
             the same as the length of all tuples in `values`
-        values : list-like of tuples
+        values : ndarray[object] of tuples
             must be sorted and all have the same length.  Should be the set of
             the MultiIndex's values.  Needed only if `method` is not None
         method: string
             "backfill" or "pad"
-        limit: int, optional
+        limit: int or None
             if provided, limit the number of fills to this value
 
         Returns
         -------
         np.ndarray[int64_t, ndim=1] of the indexer of `target` into `values`,
         filled with the `method` (and optionally `limit`) specified
         """
-        if method is None:
-            return self.get_indexer_no_fill(target)
-
         assert method in ("backfill", "pad")
         cdef:
             int64_t i, j, next_code
@@ -658,8 +655,8 @@ cdef class BaseMultiIndexCodesEngine:
             ndarray[int64_t, ndim=1] new_codes, new_target_codes
             ndarray[int64_t, ndim=1] sorted_indexer
 
-        target_order = np.argsort(target.values).astype('int64')
-        target_values = target.values[target_order]
+        target_order = np.argsort(target).astype('int64')
+        target_values = target[target_order]
         num_values, num_target_values = len(values), len(target_values)
         new_codes, new_target_codes = (
             np.empty((num_values,)).astype('int64'),
@@ -718,7 +715,7 @@ cdef class BaseMultiIndexCodesEngine:
 
         return self._base.get_loc(self, lab_int)
 
-    def get_indexer_non_unique(self, object target):
+    def get_indexer_non_unique(self, ndarray target):
         # This needs to be overridden just because the default one works on
         # target._values, and target can be itself a MultiIndex.
 

pandas/_libs/ops.pyi

@@ -0,0 +1,43 @@
+from typing import (
+    Any,
+    Callable,
+)
+
+import numpy as np
+
+_BinOp = Callable[[Any, Any], Any]
+_BoolOp = Callable[[Any, Any], bool]
+
+
+def scalar_compare(
+    values: np.ndarray,  # object[:]
+    val: object,
+    op: _BoolOp,          # {operator.eq, operator.ne, ...}
+) -> np.ndarray: ...     # np.ndarray[bool]
+
+def vec_compare(
+    left: np.ndarray,   # np.ndarray[object]
+    right: np.ndarray,  # np.ndarray[object]
+    op: _BoolOp,         # {operator.eq, operator.ne, ...}
+) -> np.ndarray: ...    # np.ndarray[bool]
+
+
+def scalar_binop(
+    values: np.ndarray,   # object[:]
+    val: object,
+    op: _BinOp,           # binary operator
+) -> np.ndarray: ...
+
+
+def vec_binop(
+    left: np.ndarray,   # object[:]
+    right: np.ndarray,  # object[:]
+    op: _BinOp,         # binary operator
+) -> np.ndarray: ...
+
+
+def maybe_convert_bool(
+    arr: np.ndarray,  # np.ndarray[object]
+    true_values=...,
+    false_values=...
+) -> np.ndarray: ...

pandas/_libs/ops_dispatch.pyi

@@ -0,0 +1,5 @@
+import numpy as np
+
+def maybe_dispatch_ufunc_to_dunder_op(
+    self, ufunc: np.ufunc, method: str, *inputs, **kwargs
+): ...

pandas/_libs/reshape.pyi

@@ -0,0 +1,19 @@
+import numpy as np
+
+def unstack(
+    values: np.ndarray,    # reshape_t[:, :]
+    mask: np.ndarray,      # const uint8_t[:]
+    stride: int,
+    length: int,
+    width: int,
+    new_values: np.ndarray,  # reshape_t[:, :]
+    new_mask: np.ndarray,    # uint8_t[:, :]
+) -> None: ...
+
+
+def explode(
+    values: np.ndarray,  # np.ndarray[object]
+) -> tuple[
+    np.ndarray,  # np.ndarray[object]
+    np.ndarray,  # np.ndarray[np.int64]
+]: ...

pandas/_libs/writers.pyx

@@ -34,7 +34,7 @@ def write_csv_rows(
     data_index : ndarray
     nlevels : int
     cols : ndarray
-    writer : object
+    writer : _csv.writer
     """
     # In crude testing, N>100 yields little marginal improvement
     cdef:

pandas/core/array_algos/quantile.py

@@ -142,17 +142,10 @@ def quantile_ea_compat(
     mask = np.asarray(values.isna())
     mask = np.atleast_2d(mask)
 
-    # error: Incompatible types in assignment (expression has type "ndarray", variable
-    # has type "ExtensionArray")
-    values, fill_value = values._values_for_factorize()  # type: ignore[assignment]
-    # error: No overload variant of "atleast_2d" matches argument type "ExtensionArray"
-    values = np.atleast_2d(values)  # type: ignore[call-overload]
-
-    # error: Argument 1 to "quantile_with_mask" has incompatible type "ExtensionArray";
-    # expected "ndarray"
-    result = quantile_with_mask(
-        values, mask, fill_value, qs, interpolation, axis  # type: ignore[arg-type]
-    )
+    arr, fill_value = values._values_for_factorize()
+    arr = np.atleast_2d(arr)
+
+    result = quantile_with_mask(arr, mask, fill_value, qs, interpolation, axis)
 
     if not is_sparse(orig.dtype):
         # shape[0] should be 1 as long as EAs are 1D

pandas/core/array_algos/replace.py

@@ -152,8 +152,6 @@ def re_replacer(s):
     f = np.vectorize(re_replacer, otypes=[values.dtype])
 
     if mask is None:
-        # error: Invalid index type "slice" for "ExtensionArray"; expected type
-        # "Union[int, ndarray]"
-        values[:] = f(values)  # type: ignore[index]
+        values[:] = f(values)
     else:
         values[mask] = f(values[mask])

pandas/core/array_algos/take.py

@@ -177,9 +177,6 @@ def take_1d(
 
     Note: similarly to `take_nd`, this function assumes that the indexer is
     a valid(ated) indexer with no out of bound indices.
-
-    TODO(ArrayManager): mainly useful for ArrayManager, otherwise can potentially
-    be removed again if we don't end up with ArrayManager.
     """
     if not isinstance(arr, np.ndarray):
         # ExtensionArray -> dispatch to their method

pandas/core/arrays/base.py

@@ -326,7 +326,7 @@ def __getitem__(
         """
         raise AbstractMethodError(self)
 
-    def __setitem__(self, key: Union[int, np.ndarray], value: Any) -> None:
+    def __setitem__(self, key: Union[int, slice, np.ndarray], value: Any) -> None:
         """
         Set one or more values inplace.
 

pandas/core/arrays/datetimes.py

@@ -7,9 +7,11 @@
     tzinfo,
 )
 from typing import (
+    TYPE_CHECKING,
     Optional,
     Union,
     cast,
+    overload,
 )
 import warnings
 
@@ -79,6 +81,9 @@
     Tick,
 )
 
+if TYPE_CHECKING:
+    from typing import Literal
+
 _midnight = time(0, 0)
 
 
@@ -1909,6 +1914,20 @@ def std(
 # Constructor Helpers
 
 
+@overload
+def sequence_to_datetimes(
+    data, allow_object: Literal[False] = ..., require_iso8601: bool = ...
+) -> DatetimeArray:
+    ...
+
+
+@overload
+def sequence_to_datetimes(
+    data, allow_object: Literal[True] = ..., require_iso8601: bool = ...
+) -> Union[np.ndarray, DatetimeArray]:
+    ...
+
+
 def sequence_to_datetimes(
     data, allow_object: bool = False, require_iso8601: bool = False
 ) -> Union[np.ndarray, DatetimeArray]:

pandas/core/arrays/string_arrow.py

@@ -9,6 +9,7 @@
     Tuple,
     Type,
     Union,
+    cast,
 )
 
 import numpy as np
@@ -485,7 +486,7 @@ def _cmp_method(self, other, op):
         # TODO(ARROW-9429): Add a .to_numpy() to ChunkedArray
         return BooleanArray._from_sequence(result.to_pandas().values)
 
-    def __setitem__(self, key: Union[int, np.ndarray], value: Any) -> None:
+    def __setitem__(self, key: Union[int, slice, np.ndarray], value: Any) -> None:
         """Set one or more values inplace.
 
         Parameters
@@ -509,6 +510,8 @@ def __setitem__(self, key: Union[int, np.ndarray], value: Any) -> None:
         key = check_array_indexer(self, key)
 
         if is_integer(key):
+            key = cast(int, key)
+
             if not is_scalar(value):
                 raise ValueError("Must pass scalars with scalar indexer")
             elif isna(value):
@@ -518,8 +521,7 @@ def __setitem__(self, key: Union[int, np.ndarray], value: Any) -> None:
 
             # Slice data and insert in-between
             new_data = [
-                # error: Slice index must be an integer or None
-                *self._data[0:key].chunks,  # type: ignore[misc]
+                *self._data[0:key].chunks,
                 pa.array([value], type=pa.string()),
                 *self._data[(key + 1) :].chunks,
             ]
@@ -530,11 +532,11 @@ def __setitem__(self, key: Union[int, np.ndarray], value: Any) -> None:
             #       This is probably extremely slow.
 
             # Convert all possible input key types to an array of integers
-            if is_bool_dtype(key):
+            if isinstance(key, slice):
+                key_array = np.array(range(len(self))[key])
+            elif is_bool_dtype(key):
                 # TODO(ARROW-9430): Directly support setitem(booleans)
                 key_array = np.argwhere(key).flatten()
-            elif isinstance(key, slice):
-                key_array = np.array(range(len(self))[key])
             else:
                 # TODO(ARROW-9431): Directly support setitem(integers)
                 key_array = np.asanyarray(key)

pandas/core/arrays/timedeltas.py

@@ -42,7 +42,6 @@
 from pandas.core.dtypes.common import (
     DT64NS_DTYPE,
     TD64NS_DTYPE,
-    is_categorical_dtype,
     is_dtype_equal,
     is_float_dtype,
     is_integer_dtype,
@@ -53,7 +52,10 @@
     pandas_dtype,
 )
 from pandas.core.dtypes.dtypes import DatetimeTZDtype
-from pandas.core.dtypes.generic import ABCMultiIndex
+from pandas.core.dtypes.generic import (
+    ABCCategorical,
+    ABCMultiIndex,
+)
 from pandas.core.dtypes.missing import isna
 
 from pandas.core import nanops
@@ -970,7 +972,7 @@ def sequence_to_td64ns(
     elif not isinstance(data, (np.ndarray, ExtensionArray)):
         # GH#24539 e.g. xarray, dask object
         data = np.asarray(data)
-    elif is_categorical_dtype(data.dtype):
+    elif isinstance(data, ABCCategorical):
         data = data.categories.take(data.codes, fill_value=NaT)._values
         copy = False