Merge remote-tracking branch 'upstream/master' into fix-issue-32621

Author: chrispe

asv_bench/benchmarks/multiindex_object.py

@@ -74,10 +74,38 @@ def setup(self):
             ],
             dtype=object,
         )
+        self.other_mi_many_mismatches = MultiIndex.from_tuples(
+            [
+                (-7, 41),
+                (-2, 3),
+                (-0.7, 5),
+                (0, 0),
+                (0, 1.5),
+                (0, 340),
+                (0, 1001),
+                (1, -4),
+                (1, 20),
+                (1, 1040),
+                (432, -5),
+                (432, 17),
+                (439, 165.5),
+                (998, -4),
+                (998, 24065),
+                (999, 865.2),
+                (999, 1000),
+                (1045, -843),
+            ]
+        )
 
     def time_get_indexer(self):
         self.mi_int.get_indexer(self.obj_index)
 
+    def time_get_indexer_and_backfill(self):
+        self.mi_int.get_indexer(self.other_mi_many_mismatches, method="backfill")
+
+    def time_get_indexer_and_pad(self):
+        self.mi_int.get_indexer(self.other_mi_many_mismatches, method="pad")
+
     def time_is_monotonic(self):
         self.mi_int.is_monotonic
 

asv_bench/benchmarks/rolling.py

@@ -165,4 +165,26 @@ def peakmem_fixed(self):
             self.roll.max()
 
 
+class ForwardWindowMethods:
+    params = (
+        ["DataFrame", "Series"],
+        [10, 1000],
+        ["int", "float"],
+        ["median", "mean", "max", "min", "kurt", "sum"],
+    )
+    param_names = ["constructor", "window_size", "dtype", "method"]
+
+    def setup(self, constructor, window_size, dtype, method):
+        N = 10 ** 5
+        arr = np.random.random(N).astype(dtype)
+        indexer = pd.api.indexers.FixedForwardWindowIndexer(window_size=window_size)
+        self.roll = getattr(pd, constructor)(arr).rolling(window=indexer)
+
+    def time_rolling(self, constructor, window_size, dtype, method):
+        getattr(self.roll, method)()
+
+    def peakmem_rolling(self, constructor, window_size, dtype, method):
+        getattr(self.roll, method)()
+
+
 from .pandas_vb_common import setup  # noqa: F401 isort:skip

doc/source/reference/window.rst

@@ -85,3 +85,4 @@ Base class for defining custom window boundaries.
    :toctree: api/
 
    api.indexers.BaseIndexer
+   api.indexers.FixedForwardWindowIndexer

doc/source/user_guide/computation.rst

@@ -571,6 +571,20 @@ and we want to use an expanding window where ``use_expanding`` is ``True`` other
    3     3.0
    4    10.0
 
+.. versionadded:: 1.1
+
+For some problems knowledge of the future is available for analysis. For example, this occurs when
+each data point is a full time series read from an experiment, and the task is to extract underlying
+conditions. In these cases it can be useful to perform forward-looking rolling window computations.
+:func:`FixedForwardWindowIndexer <pandas.api.indexers.FixedForwardWindowIndexer>` class is available for this purpose.
+This :func:`BaseIndexer <pandas.api.indexers.BaseIndexer>` subclass implements a closed fixed-width
+forward-looking rolling window, and we can use it as follows:
+
+.. ipython:: ipython
+
+   from pandas.api.indexers import FixedForwardWindowIndexer
+   indexer = FixedForwardWindowIndexer(window_size=2)
+   df.rolling(indexer, min_periods=1).sum()
 
 .. _stats.rolling_window.endpoints:
 

doc/source/whatsnew/v1.1.0.rst

@@ -109,6 +109,7 @@ Other API changes
 - ``loc`` lookups with an object-dtype :class:`Index` and an integer key will now raise ``KeyError`` instead of ``TypeError`` when key is missing (:issue:`31905`)
 - Using a :func:`pandas.api.indexers.BaseIndexer` with ``std``, ``var``, ``count``, ``skew``, ``cov``, ``corr`` will now raise a ``NotImplementedError`` (:issue:`32865`)
 - Using a :func:`pandas.api.indexers.BaseIndexer` with ``min``, ``max`` will now return correct results for any monotonic :func:`pandas.api.indexers.BaseIndexer` descendant (:issue:`32865`)
+- Added a :func:`pandas.api.indexers.FixedForwardWindowIndexer` class to support forward-looking windows during ``rolling`` operations.
 -
 
 Backwards incompatible API changes
@@ -120,6 +121,67 @@ Backwards incompatible API changes
   Previously a ``UnsupportedFunctionCall`` was raised (``AssertionError`` if ``min_count`` passed into :meth:`~DataFrameGroupby.median`) (:issue:`31485`)
 - :meth:`DataFrame.at` and :meth:`Series.at` will raise a ``TypeError`` instead of a ``ValueError`` if an incompatible key is passed, and ``KeyError`` if a missing key is passed, matching the behavior of ``.loc[]`` (:issue:`31722`)
 - Passing an integer dtype other than ``int64`` to ``np.array(period_index, dtype=...)`` will now raise ``TypeError`` instead of incorrectly using ``int64`` (:issue:`32255`)
+
+``MultiIndex.get_indexer`` interprets `method` argument differently
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This restores the behavior of :meth:`MultiIndex.get_indexer` with ``method='backfill'`` or ``method='pad'`` to the behavior before pandas 0.23.0. In particular, MultiIndexes are treated as a list of tuples and padding or backfilling is done with respect to the ordering of these lists of tuples (:issue:`29896`).
+
+As an example of this, given:
+
+.. ipython:: python
+
+        df = pd.DataFrame({
+            'a': [0, 0, 0, 0],
+            'b': [0, 2, 3, 4],
+            'c': ['A', 'B', 'C', 'D'],
+        }).set_index(['a', 'b'])
+        mi_2 = pd.MultiIndex.from_product([[0], [-1, 0, 1, 3, 4, 5]])
+
+The differences in reindexing ``df`` with ``mi_2`` and using ``method='backfill'`` can be seen here:
+
+*pandas >= 0.23, < 1.1.0*:
+
+.. code-block:: ipython
+
+    In [1]: df.reindex(mi_2, method='backfill')
+    Out[1]:
+          c
+    0 -1  A
+       0  A
+       1  D
+       3  A
+       4  A
+       5  C
+
+*pandas <0.23, >= 1.1.0*
+
+.. ipython:: python
+
+        df.reindex(mi_2, method='backfill')
+
+And the differences in reindexing ``df`` with ``mi_2`` and using ``method='pad'`` can be seen here:
+
+*pandas >= 0.23, < 1.1.0*
+
+.. code-block:: ipython
+
+    In [1]: df.reindex(mi_2, method='pad')
+    Out[1]:
+            c
+    0 -1  NaN
+       0  NaN
+       1    D
+       3  NaN
+       4    A
+       5    C
+
+*pandas < 0.23, >= 1.1.0*
+
+.. ipython:: python
+
+        df.reindex(mi_2, method='pad')
+
 -
 
 .. _whatsnew_110.api_breaking.indexing_raises_key_errors:
@@ -274,7 +336,7 @@ Deprecations
   version 1.1. All other arguments should be given as keyword
   arguments (:issue:`27573`).
 
--
+- :func:`pandas.api.types.is_categorical` is deprecated and will be removed in a future version; use `:func:pandas.api.types.is_categorical_dtype` instead (:issue:`33385`)
 
 .. ---------------------------------------------------------------------------
 
@@ -434,6 +496,7 @@ I/O
 - Bug in :meth:`read_sas` was raising an ``AttributeError`` when reading files from Google Cloud Storage (issue:`33069`)
 - Bug in :meth:`DataFrame.to_sql` where an ``AttributeError`` was raised when saving an out of bounds date (:issue:`26761`)
 - Bug in :meth:`read_excel` did not correctly handle multiple embedded spaces in OpenDocument text cells. (:issue:`32207`)
+- Bug in :meth:`read_json` was raising ``TypeError`` when reading a list of booleans into a Series. (:issue:`31464`)
 
 Plotting
 ^^^^^^^^

environment.yml

@@ -86,7 +86,7 @@ dependencies:
   - lxml
 
   # pd.read_excel, DataFrame.to_excel, pd.ExcelWriter, pd.ExcelFile
-  - openpyxl<=3.0.1
+  - openpyxl
   - xlrd
   - xlsxwriter
   - xlwt

pandas/_libs/index.pyx

@@ -612,25 +612,113 @@ cdef class BaseMultiIndexCodesEngine:
                        in zip(self.levels, zip(*target))]
         return self._codes_to_ints(np.array(level_codes, dtype='uint64').T)
 
-    def get_indexer(self, object target, object method=None,
-                    object limit=None):
+    def get_indexer_no_fill(self, object target) -> np.ndarray:
+        """
+        Returns an array giving the positions of each value of `target` in
+        `self.values`, where -1 represents a value in `target` which does not
+        appear in `self.values`
+
+        Parameters
+        ----------
+        target : list-like of keys
+            Each key is a tuple, with a label for each level of the index
+
+        Returns
+        -------
+        np.ndarray[int64_t, ndim=1] of the indexer of `target` into
+        `self.values`
+        """
         lab_ints = self._extract_level_codes(target)
+        return self._base.get_indexer(self, lab_ints)
 
-        # All methods (exact, backfill, pad) directly map to the respective
-        # methods of the underlying (integers) index...
-        if method is not None:
-            # but underlying backfill and pad methods require index and keys
-            # to be sorted. The index already is (checked in
-            # Index._get_fill_indexer), sort (integer representations of) keys:
-            order = np.argsort(lab_ints)
-            lab_ints = lab_ints[order]
-            indexer = (getattr(self._base, f'get_{method}_indexer')
-                       (self, lab_ints, limit=limit))
-            indexer = indexer[order]
-        else:
-            indexer = self._base.get_indexer(self, lab_ints)
+    def get_indexer(self, object target, object values = None,
+                    object method = None, object limit = None) -> np.ndarray:
+        """
+        Returns an array giving the positions of each value of `target` in
+        `values`, where -1 represents a value in `target` which does not
+        appear in `values`
 
-        return indexer
+        If `method` is "backfill" then the position for a value in `target`
+        which does not appear in `values` is that of the next greater value
+        in `values` (if one exists), and -1 if there is no such value.
+
+        Similarly, if the method is "pad" then the position for a value in
+        `target` which does not appear in `values` is that of the next smaller
+        value in `values` (if one exists), and -1 if there is no such value.
+
+        Parameters
+        ----------
+        target: list-like of tuples
+            need not be sorted, but all must have the same length, which must be
+            the same as the length of all tuples in `values`
+        values : list-like of tuples
+            must be sorted and all have the same length.  Should be the set of
+            the MultiIndex's values.  Needed only if `method` is not None
+        method: string
+            "backfill" or "pad"
+        limit: int, optional
+            if provided, limit the number of fills to this value
+
+        Returns
+        -------
+        np.ndarray[int64_t, ndim=1] of the indexer of `target` into `values`,
+        filled with the `method` (and optionally `limit`) specified
+        """
+        if method is None:
+            return self.get_indexer_no_fill(target)
+
+        assert method in ("backfill", "pad")
+        cdef:
+            int64_t i, j, next_code
+            int64_t num_values, num_target_values
+            ndarray[int64_t, ndim=1] target_order
+            ndarray[object, ndim=1] target_values
+            ndarray[int64_t, ndim=1] new_codes, new_target_codes
+            ndarray[int64_t, ndim=1] sorted_indexer
+
+        target_order = np.argsort(target.values).astype('int64')
+        target_values = target.values[target_order]
+        num_values, num_target_values = len(values), len(target_values)
+        new_codes, new_target_codes = (
+            np.empty((num_values,)).astype('int64'),
+            np.empty((num_target_values,)).astype('int64'),
+        )
+
+        # `values` and `target_values` are both sorted, so we walk through them
+        # and memoize the (ordered) set of indices in the (implicit) merged-and
+        # sorted list of the two which belong to each of them
+        # the effect of this is to create a factorization for the (sorted)
+        # merger of the index values, where `new_codes` and `new_target_codes`
+        # are the subset of the factors which appear in `values` and `target`,
+        # respectively
+        i, j, next_code = 0, 0, 0
+        while i < num_values and j < num_target_values:
+            val, target_val = values[i], target_values[j]
+            if val <= target_val:
+                new_codes[i] = next_code
+                i += 1
+            if target_val <= val:
+                new_target_codes[j] = next_code
+                j += 1
+            next_code += 1
+
+        # at this point, at least one should have reached the end
+        # the remaining values of the other should be added to the end
+        assert i == num_values or j == num_target_values
+        while i < num_values:
+            new_codes[i] = next_code
+            i += 1
+            next_code += 1
+        while j < num_target_values:
+            new_target_codes[j] = next_code
+            j += 1
+            next_code += 1
+
+        # get the indexer, and undo the sorting of `target.values`
+        sorted_indexer = (
+            algos.backfill if method == "backfill" else algos.pad
+        )(new_codes, new_target_codes, limit=limit).astype('int64')
+        return sorted_indexer[np.argsort(target_order)]
 
     def get_loc(self, object key):
         if is_definitely_invalid_key(key):

pandas/api/indexers/__init__.py

@@ -3,6 +3,6 @@
 """
 
 from pandas.core.indexers import check_array_indexer
-from pandas.core.window.indexers import BaseIndexer
+from pandas.core.window.indexers import BaseIndexer, FixedForwardWindowIndexer
 
-__all__ = ["check_array_indexer", "BaseIndexer"]
+__all__ = ["check_array_indexer", "BaseIndexer", "FixedForwardWindowIndexer"]

pandas/core/dtypes/common.py

@@ -125,7 +125,7 @@ def ensure_categorical(arr):
     cat_arr : The original array cast as a Categorical. If it already
               is a Categorical, we return as is.
     """
-    if not is_categorical(arr):
+    if not is_categorical_dtype(arr.dtype):
         from pandas import Categorical
 
         arr = Categorical(arr)
@@ -360,6 +360,12 @@ def is_categorical(arr) -> bool:
     >>> is_categorical(pd.CategoricalIndex([1, 2, 3]))
     True
     """
+    warnings.warn(
+        "is_categorical is deprecated and will be removed in a future version.  "
+        "Use is_categorical_dtype instead",
+        FutureWarning,
+        stacklevel=2,
+    )
     return isinstance(arr, ABCCategorical) or is_categorical_dtype(arr)
 
 
@@ -1458,7 +1464,7 @@ def is_extension_type(arr) -> bool:
         stacklevel=2,
     )
 
-    if is_categorical(arr):
+    if is_categorical_dtype(arr):
         return True
     elif is_sparse(arr):
         return True

pandas/core/dtypes/dtypes.py

@@ -337,7 +337,6 @@ def _from_values_or_dtype(
         >>> pd.CategoricalDtype._from_values_or_dtype(c, dtype=dtype2)
         CategoricalDtype(categories=['x', 'y'], ordered=False)
         """
-        from pandas.core.dtypes.common import is_categorical
 
         if dtype is not None:
             # The dtype argument takes precedence over values.dtype (if any)
@@ -352,7 +351,7 @@ def _from_values_or_dtype(
                 )
             elif not isinstance(dtype, CategoricalDtype):
                 raise ValueError(f"Cannot not construct CategoricalDtype from {dtype}")
-        elif is_categorical(values):
+        elif cls.is_dtype(values):
             # If no "dtype" was passed, use the one from "values", but honor
             # the "ordered" and "categories" arguments
             dtype = values.dtype._from_categorical_dtype(