Merge remote-tracking branch 'upstream/master' into STY-repr-batch-3

Author: MomIsBestFriend

ci/print_skipped.py

@@ -5,7 +5,7 @@
 
 def main(filename):
     if not os.path.isfile(filename):
-        raise RuntimeError(f"Could not find junit file {filename!r}")
+        raise RuntimeError(f"Could not find junit file {repr(filename)}")
 
     tree = et.parse(filename)
     root = tree.getroot()

doc/redirects.csv

@@ -360,7 +360,6 @@ generated/pandas.DataFrame.from_records,../reference/api/pandas.DataFrame.from_r
 generated/pandas.DataFrame.ge,../reference/api/pandas.DataFrame.ge
 generated/pandas.DataFrame.get,../reference/api/pandas.DataFrame.get
 generated/pandas.DataFrame.get_value,../reference/api/pandas.DataFrame.get_value
-generated/pandas.DataFrame.get_values,../reference/api/pandas.DataFrame.get_values
 generated/pandas.DataFrame.groupby,../reference/api/pandas.DataFrame.groupby
 generated/pandas.DataFrame.gt,../reference/api/pandas.DataFrame.gt
 generated/pandas.DataFrame.head,../reference/api/pandas.DataFrame.head
@@ -643,7 +642,6 @@ generated/pandas.Index.get_level_values,../reference/api/pandas.Index.get_level_
 generated/pandas.Index.get_loc,../reference/api/pandas.Index.get_loc
 generated/pandas.Index.get_slice_bound,../reference/api/pandas.Index.get_slice_bound
 generated/pandas.Index.get_value,../reference/api/pandas.Index.get_value
-generated/pandas.Index.get_values,../reference/api/pandas.Index.get_values
 generated/pandas.Index.groupby,../reference/api/pandas.Index.groupby
 generated/pandas.Index.has_duplicates,../reference/api/pandas.Index.has_duplicates
 generated/pandas.Index.hasnans,../reference/api/pandas.Index.hasnans
@@ -1044,7 +1042,6 @@ generated/pandas.Series.from_csv,../reference/api/pandas.Series.from_csv
 generated/pandas.Series.ge,../reference/api/pandas.Series.ge
 generated/pandas.Series.get,../reference/api/pandas.Series.get
 generated/pandas.Series.get_value,../reference/api/pandas.Series.get_value
-generated/pandas.Series.get_values,../reference/api/pandas.Series.get_values
 generated/pandas.Series.groupby,../reference/api/pandas.Series.groupby
 generated/pandas.Series.gt,../reference/api/pandas.Series.gt
 generated/pandas.Series.hasnans,../reference/api/pandas.Series.hasnans

doc/source/reference/frame.rst

@@ -30,7 +30,6 @@ Attributes and underlying data
    DataFrame.dtypes
    DataFrame.select_dtypes
    DataFrame.values
-   DataFrame.get_values
    DataFrame.axes
    DataFrame.ndim
    DataFrame.size

doc/source/reference/indexing.rst

@@ -159,7 +159,6 @@ Selecting
    Index.get_loc
    Index.get_slice_bound
    Index.get_value
-   Index.get_values
    Index.isin
    Index.slice_indexer
    Index.slice_locs

doc/source/reference/series.rst

@@ -53,7 +53,6 @@ Conversion
    Series.to_period
    Series.to_timestamp
    Series.to_list
-   Series.get_values
    Series.__array__
 
 Indexing, iteration

pandas/_libs/missing.pyx

@@ -289,7 +289,8 @@ cdef inline bint is_null_period(v):
 def _create_binary_propagating_op(name, divmod=False):
 
     def method(self, other):
-        if other is C_NA or isinstance(other, str) or isinstance(other, numbers.Number):
+        if (other is C_NA or isinstance(other, str)
+                or isinstance(other, (numbers.Number, np.bool_))):
             if divmod:
                 return NA, NA
             else:

pandas/_libs/reduction.pyx

@@ -26,14 +26,6 @@ cdef _check_result_array(object obj, Py_ssize_t cnt):
         raise ValueError('Function does not reduce')
 
 
-cdef bint _is_sparse_array(object obj):
-    # TODO can be removed one SparseArray.values is removed (GH26421)
-    if hasattr(obj, '_subtyp'):
-        if obj._subtyp == 'sparse_array':
-            return True
-    return False
-
-
 cdef class Reducer:
     """
     Performs generic reduction operation on a C or Fortran-contiguous ndarray
@@ -404,8 +396,7 @@ cdef class SeriesGrouper(_BaseGrouper):
 cdef inline _extract_result(object res, bint squeeze=True):
     """ extract the result object, it might be a 0-dim ndarray
         or a len-1 0-dim, or a scalar """
-    if (not _is_sparse_array(res) and hasattr(res, 'values')
-            and util.is_array(res.values)):
+    if hasattr(res, 'values') and util.is_array(res.values):
         res = res.values
     if util.is_array(res):
         if res.ndim == 0:

pandas/compat/__init__.py

@@ -98,11 +98,11 @@ def is_platform_32bit() -> bool:
 
 def _import_lzma():
     """
-    Attempts to import the lzma module.
+    Importing the `lzma` module.
 
     Warns
     -----
-    When the lzma module is not available.
+    When the `lzma` module is not available.
     """
     try:
         import lzma
@@ -119,22 +119,22 @@ def _import_lzma():
 
 def _get_lzma_file(lzma):
     """
-    Attempting to get the lzma.LZMAFile class.
+    Importing the `LZMAFile` class from the `lzma` module.
 
     Returns
     -------
     class
-        The lzma.LZMAFile class.
+        The `LZMAFile` class from the `lzma` module.
 
     Raises
     ------
     RuntimeError
-        If the module lzma was not imported correctly, or didn't exist.
+        If the `lzma` module was not imported correctly, or didn't exist.
     """
     if lzma is None:
         raise RuntimeError(
             "lzma module not available. "
-            "A Python re-install with the proper "
-            "dependencies might be required to solve this issue."
+            "A Python re-install with the proper dependencies, "
+            "might be required to solve this issue."
         )
     return lzma.LZMAFile

pandas/core/arrays/boolean.py

@@ -1,10 +1,10 @@
 import numbers
-from typing import TYPE_CHECKING, Type
+from typing import TYPE_CHECKING, Any, Tuple, Type
 import warnings
 
 import numpy as np
 
-from pandas._libs import lib
+from pandas._libs import lib, missing as libmissing
 from pandas.compat import set_function_name
 
 from pandas.core.dtypes.base import ExtensionDtype
@@ -61,13 +61,13 @@ class BooleanDtype(ExtensionDtype):
     @property
     def na_value(self) -> "Scalar":
         """
-        BooleanDtype uses :attr:`numpy.nan` as the missing NA value.
+        BooleanDtype uses :attr:`pandas.NA` as the missing NA value.
 
         .. warning::
 
            `na_value` may change in a future release.
         """
-        return np.nan
+        return libmissing.NA
 
     @property
     def type(self) -> Type:
@@ -223,7 +223,7 @@ class BooleanArray(ExtensionArray, ExtensionOpsMixin):
 
     >>> pd.array([True, False, None], dtype="boolean")
     <BooleanArray>
-    [True, False, NaN]
+    [True, False, NA]
     Length: 3, dtype: boolean
     """
 
@@ -262,17 +262,17 @@ def _from_sequence(cls, scalars, dtype=None, copy: bool = False):
         values, mask = coerce_to_array(scalars, copy=copy)
         return BooleanArray(values, mask)
 
+    def _values_for_factorize(self) -> Tuple[np.ndarray, Any]:
+        data = self._data.astype("int8")
+        data[self._mask] = -1
+        return data, -1
+
     @classmethod
     def _from_factorized(cls, values, original: "BooleanArray"):
         return cls._from_sequence(values, dtype=original.dtype)
 
     def _formatter(self, boxed=False):
-        def fmt(x):
-            if isna(x):
-                return "NaN"
-            return str(x)
-
-        return fmt
+        return str
 
     def __getitem__(self, item):
         if is_integer(item):
@@ -281,25 +281,29 @@ def __getitem__(self, item):
             return self._data[item]
         return type(self)(self._data[item], self._mask[item])
 
-    def _coerce_to_ndarray(self, force_bool: bool = False):
+    def _coerce_to_ndarray(self, dtype=None, na_value: "Scalar" = libmissing.NA):
         """
         Coerce to an ndarary of object dtype or bool dtype (if force_bool=True).
 
         Parameters
         ----------
-        force_bool : bool, default False
-            If True, return bool array or raise error if not possible (in
-            presence of missing values)
+        dtype : dtype, default object
+            The numpy dtype to convert to
+        na_value : scalar, optional
+             Scalar missing value indicator to use in numpy array. Defaults
+             to the native missing value indicator of this array (pd.NA).
         """
-        if force_bool:
+        if dtype is None:
+            dtype = object
+        if is_bool_dtype(dtype):
             if not self.isna().any():
                 return self._data
             else:
                 raise ValueError(
                     "cannot convert to bool numpy array in presence of missing values"
                 )
-        data = self._data.astype(object)
-        data[self._mask] = self._na_value
+        data = self._data.astype(dtype)
+        data[self._mask] = na_value
         return data
 
     __array_priority__ = 1000  # higher than ndarray so ops dispatch to us
@@ -309,15 +313,8 @@ def __array__(self, dtype=None):
         the array interface, return my values
         We return an object array here to preserve our scalar values
         """
-        if dtype is not None:
-            if is_bool_dtype(dtype):
-                return self._coerce_to_ndarray(force_bool=True)
-            # TODO can optimize this to not go through object dtype for
-            # numeric dtypes
-            arr = self._coerce_to_ndarray()
-            return arr.astype(dtype, copy=False)
         # by default (no dtype specified), return an object array
-        return self._coerce_to_ndarray()
+        return self._coerce_to_ndarray(dtype=dtype)
 
     def __arrow_array__(self, type=None):
         """
@@ -483,8 +480,17 @@ def astype(self, dtype, copy=True):
             return IntegerArray(
                 self._data.astype(dtype.numpy_dtype), self._mask.copy(), copy=False
             )
+        # for integer, error if there are missing values
+        if is_integer_dtype(dtype):
+            if self.isna().any():
+                raise ValueError("cannot convert NA to integer")
+        # for float dtype, ensure we use np.nan before casting (numpy cannot
+        # deal with pd.NA)
+        na_value = self._na_value
+        if is_float_dtype(dtype):
+            na_value = np.nan
         # coerce
-        data = self._coerce_to_ndarray()
+        data = self._coerce_to_ndarray(na_value=na_value)
         return astype_nansafe(data, dtype, copy=None)
 
     def value_counts(self, dropna=True):
@@ -594,8 +600,6 @@ def logical_method(self, other):
 
     @classmethod
     def _create_comparison_method(cls, op):
-        op_name = op.__name__
-
         def cmp_method(self, other):
 
             if isinstance(other, (ABCDataFrame, ABCSeries, ABCIndexClass)):
@@ -617,21 +621,26 @@ def cmp_method(self, other):
                 if len(self) != len(other):
                     raise ValueError("Lengths must match to compare")
 
-            # numpy will show a DeprecationWarning on invalid elementwise
-            # comparisons, this will raise in the future
-            with warnings.catch_warnings():
-                warnings.filterwarnings("ignore", "elementwise", FutureWarning)
-                with np.errstate(all="ignore"):
-                    result = op(self._data, other)
-
-            # nans propagate
-            if mask is None:
-                mask = self._mask
+            if other is libmissing.NA:
+                # numpy does not handle pd.NA well as "other" scalar (it returns
+                # a scalar False instead of an array)
+                result = np.zeros_like(self._data)
+                mask = np.ones_like(self._data)
             else:
-                mask = self._mask | mask
+                # numpy will show a DeprecationWarning on invalid elementwise
+                # comparisons, this will raise in the future
+                with warnings.catch_warnings():
+                    warnings.filterwarnings("ignore", "elementwise", FutureWarning)
+                    with np.errstate(all="ignore"):
+                        result = op(self._data, other)
+
+                # nans propagate
+                if mask is None:
+                    mask = self._mask.copy()
+                else:
+                    mask = self._mask | mask
 
-            result[mask] = op_name == "ne"
-            return BooleanArray(result, np.zeros(len(result), dtype=bool), copy=False)
+            return BooleanArray(result, mask, copy=False)
 
         name = "__{name}__".format(name=op.__name__)
         return set_function_name(cmp_method, name, cls)
@@ -643,7 +652,7 @@ def _reduce(self, name, skipna=True, **kwargs):
         # coerce to a nan-aware float if needed
         if mask.any():
             data = self._data.astype("float64")
-            data[mask] = self._na_value
+            data[mask] = np.nan
 
         op = getattr(nanops, "nan" + name)
         result = op(data, axis=0, skipna=skipna, mask=mask, **kwargs)

pandas/core/arrays/categorical.py

@@ -302,9 +302,7 @@ class Categorical(ExtensionArray, PandasObject):
     __array_priority__ = 1000
     _dtype = CategoricalDtype(ordered=False)
     # tolist is not actually deprecated, just suppressed in the __dir__
-    _deprecations = PandasObject._deprecations | frozenset(
-        ["tolist", "itemsize", "get_values"]
-    )
+    _deprecations = PandasObject._deprecations | frozenset(["tolist", "itemsize"])
     _typ = "categorical"
 
     def __init__(
@@ -1461,29 +1459,18 @@ def value_counts(self, dropna=True):
 
         return Series(count, index=CategoricalIndex(ix), dtype="int64")
 
-    def get_values(self):
+    def _internal_get_values(self):
         """
         Return the values.
 
-        .. deprecated:: 0.25.0
-
         For internal compatibility with pandas formatting.
 
         Returns
         -------
-        numpy.array
+        np.ndarray or Index
             A numpy array of the same dtype as categorical.categories.dtype or
             Index if datetime / periods.
         """
-        warn(
-            "The 'get_values' method is deprecated and will be removed in a "
-            "future version",
-            FutureWarning,
-            stacklevel=2,
-        )
-        return self._internal_get_values()
-
-    def _internal_get_values(self):
         # if we are a datetime and period index, return Index to keep metadata
         if needs_i8_conversion(self.categories):
             return self.categories.take(self._codes, fill_value=np.nan)