Merge branch 'shiny-new-feature' of https://github.com/EzraBrauner/pandas into shiny-new-feature

Author: EzraBrauner

doc/source/conf.py

@@ -225,11 +225,24 @@
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
 # documentation.
+
+switcher_version = version
+if ".dev" in version:
+    switcher_version = "dev"
+elif "rc" in version:
+    switcher_version = version.split("rc")[0] + " (rc)"
+
 html_theme_options = {
     "external_links": [],
     "github_url": "https://github.com/pandas-dev/pandas",
     "twitter_url": "https://twitter.com/pandas_dev",
     "google_analytics_id": "UA-27880019-2",
+    "navbar_end": ["version-switcher", "navbar-icon-links"],
+    "switcher": {
+        "json_url": "https://pandas.pydata.org/versions.json",
+        "url_template": "https://pandas.pydata.org/{version}/",
+        "version_match": switcher_version,
+    },
 }
 
 # Add any paths that contain custom themes here, relative to this directory.

doc/source/development/contributing_codebase.rst

@@ -490,8 +490,7 @@ Writing tests
 All tests should go into the ``tests`` subdirectory of the specific package.
 This folder contains many current examples of tests, and we suggest looking to these for
 inspiration.  If your test requires working with files or
-network connectivity, there is more information on the `testing page
-<https://github.com/pandas-dev/pandas/wiki/Testing>`_ of the wiki.
+network connectivity, there is more information on the :wiki:`Testing` of the wiki.
 
 The ``pandas._testing`` module has many special ``assert`` functions that
 make it easier to make statements about whether Series or DataFrame objects are

doc/source/development/roadmap.rst

@@ -74,8 +74,7 @@ types. This includes consistent behavior in all operations (indexing, arithmetic
 operations, comparisons, etc.). There has been discussion of eventually making
 the new semantics the default.
 
-This has been discussed at
-`github #28095 <https://github.com/pandas-dev/pandas/issues/28095>`__ (and
+This has been discussed at :issue:`28095` (and
 linked issues), and described in more detail in this
 `design doc <https://hackmd.io/@jorisvandenbossche/Sk0wMeAmB>`__.
 
@@ -129,8 +128,7 @@ We propose that it should only work with positional indexing, and the translatio
 to positions should be entirely done at a higher level.
 
 Indexing is a complicated API with many subtleties. This refactor will require care
-and attention. More details are discussed at
-https://github.com/pandas-dev/pandas/wiki/(Tentative)-rules-for-restructuring-indexing-code
+and attention. More details are discussed at :wiki:`(Tentative)-rules-for-restructuring-indexing-code`
 
 Numba-accelerated operations
 ----------------------------

doc/source/ecosystem.rst

@@ -586,7 +586,6 @@ Development tools
 While pandas repository is partially typed, the package itself doesn't expose this information for external use.
 Install pandas-stubs to enable basic type coverage of pandas API.
 
-Learn more by reading through these issues `14468 <https://github.com/pandas-dev/pandas/issues/14468>`_,
-`26766 <https://github.com/pandas-dev/pandas/issues/26766>`_, `28142 <https://github.com/pandas-dev/pandas/issues/28142>`_.
+Learn more by reading through :issue:`14468`, :issue:`26766`, :issue:`28142`.
 
 See installation and usage instructions on the `github page <https://github.com/VirtusLab/pandas-stubs>`__.

doc/source/user_guide/10min.rst

@@ -554,10 +554,8 @@ Stack
 
    tuples = list(
        zip(
-           *[
-               ["bar", "bar", "baz", "baz", "foo", "foo", "qux", "qux"],
-               ["one", "two", "one", "two", "one", "two", "one", "two"],
-           ]
+           ["bar", "bar", "baz", "baz", "foo", "foo", "qux", "qux"],
+           ["one", "two", "one", "two", "one", "two", "one", "two"],
        )
    )
    index = pd.MultiIndex.from_tuples(tuples, names=["first", "second"])

doc/source/user_guide/advanced.rst

@@ -1246,5 +1246,5 @@ This is because the (re)indexing operations above silently inserts ``NaNs`` and
 changes accordingly.  This can cause some issues when using ``numpy`` ``ufuncs``
 such as ``numpy.logical_and``.
 
-See the `this old issue <https://github.com/pandas-dev/pandas/issues/2388>`__ for a more
+See the :issue:`2388` for a more
 detailed discussion.

doc/source/user_guide/cookbook.rst

@@ -193,8 +193,7 @@ The :ref:`indexing <indexing>` docs.
 
    df[(df.AAA <= 6) & (df.index.isin([0, 2, 4]))]
 
-`Use loc for label-oriented slicing and iloc positional slicing
-<https://github.com/pandas-dev/pandas/issues/2904>`__
+Use loc for label-oriented slicing and iloc positional slicing :issue:`2904`
 
 .. ipython:: python
 
@@ -395,8 +394,7 @@ Sorting
 
    df.sort_values(by=("Labs", "II"), ascending=False)
 
-`Partial selection, the need for sortedness;
-<https://github.com/pandas-dev/pandas/issues/2995>`__
+Partial selection, the need for sortedness :issue:`2995`
 
 Levels
 ******
@@ -910,8 +908,7 @@ Valid frequency arguments to Grouper :ref:`Timeseries <timeseries.offset_aliases
 `Grouping using a MultiIndex
 <https://stackoverflow.com/questions/41483763/pandas-timegrouper-on-multiindex>`__
 
-`Using TimeGrouper and another grouping to create subgroups, then apply a custom function
-<https://github.com/pandas-dev/pandas/issues/3791>`__
+Using TimeGrouper and another grouping to create subgroups, then apply a custom function :issue:`3791`
 
 `Resampling with custom periods
 <https://stackoverflow.com/questions/15408156/resampling-with-custom-periods>`__
@@ -947,8 +944,7 @@ Depending on df construction, ``ignore_index`` may be needed
    df = pd.concat([df1, df2], ignore_index=True)
    df
 
-`Self Join of a DataFrame
-<https://github.com/pandas-dev/pandas/issues/2996>`__
+Self Join of a DataFrame :issue:`2996`
 
 .. ipython:: python
 
@@ -1070,8 +1066,7 @@ using that handle to read.
 `Inferring dtypes from a file
 <https://stackoverflow.com/questions/15555005/get-inferred-dataframe-types-iteratively-using-chunksize>`__
 
-`Dealing with bad lines
-<https://github.com/pandas-dev/pandas/issues/2886>`__
+Dealing with bad lines :issue:`2886`
 
 `Write a multi-row index CSV without writing duplicates
 <https://stackoverflow.com/questions/17349574/pandas-write-multiindex-rows-with-to-csv>`__
@@ -1205,8 +1200,7 @@ The :ref:`Excel <io.excel>` docs
 `Modifying formatting in XlsxWriter output
 <https://pbpython.com/improve-pandas-excel-output.html>`__
 
-`Loading only visible sheets
-<https://github.com/pandas-dev/pandas/issues/19842#issuecomment-892150745>`__
+Loading only visible sheets :issue:`19842#issuecomment-892150745`
 
 .. _cookbook.html:
 
@@ -1226,8 +1220,7 @@ The :ref:`HDFStores <io.hdf5>` docs
 `Simple queries with a Timestamp Index
 <https://stackoverflow.com/questions/13926089/selecting-columns-from-pandas-hdfstore-table>`__
 
-`Managing heterogeneous data using a linked multiple table hierarchy
-<https://github.com/pandas-dev/pandas/issues/3032>`__
+Managing heterogeneous data using a linked multiple table hierarchy :issue:`3032`
 
 `Merging on-disk tables with millions of rows
 <https://stackoverflow.com/questions/14614512/merging-two-tables-with-millions-of-rows-in-python/14617925#14617925>`__

doc/source/whatsnew/v0.17.1.rst

@@ -37,9 +37,7 @@ Conditional HTML formatting
 .. warning::
     This is a new feature and is under active development.
     We'll be adding features an  possibly making breaking changes in future
-    releases. Feedback is welcome_.
-
-.. _welcome: https://github.com/pandas-dev/pandas/issues/11610
+    releases. Feedback is welcome in :issue:`11610`
 
 We've added *experimental* support for conditional HTML formatting:
 the visual styling of a DataFrame based on the data.

doc/source/whatsnew/v1.3.0.rst

@@ -811,7 +811,7 @@ Other Deprecations
 - Deprecated allowing scalars to be passed to the :class:`Categorical` constructor (:issue:`38433`)
 - Deprecated constructing :class:`CategoricalIndex` without passing list-like data (:issue:`38944`)
 - Deprecated allowing subclass-specific keyword arguments in the :class:`Index` constructor, use the specific subclass directly instead (:issue:`14093`, :issue:`21311`, :issue:`22315`, :issue:`26974`)
-- Deprecated the :meth:`astype` method of datetimelike (``timedelta64[ns]``, ``datetime64[ns]``, ``Datetime64TZDtype``, ``PeriodDtype``) to convert to integer dtypes, use ``values.view(...)`` instead (:issue:`38544`)
+- Deprecated the :meth:`astype` method of datetimelike (``timedelta64[ns]``, ``datetime64[ns]``, ``Datetime64TZDtype``, ``PeriodDtype``) to convert to integer dtypes, use ``values.view(...)`` instead (:issue:`38544`). This deprecation was later reverted in pandas 1.4.0.
 - Deprecated :meth:`MultiIndex.is_lexsorted` and :meth:`MultiIndex.lexsort_depth`, use :meth:`MultiIndex.is_monotonic_increasing` instead (:issue:`32259`)
 - Deprecated keyword ``try_cast`` in :meth:`Series.where`, :meth:`Series.mask`, :meth:`DataFrame.where`, :meth:`DataFrame.mask`; cast results manually if desired (:issue:`38836`)
 - Deprecated comparison of :class:`Timestamp` objects with ``datetime.date`` objects.  Instead of e.g. ``ts <= mydate`` use ``ts <= pd.Timestamp(mydate)`` or ``ts.date() <= mydate`` (:issue:`36131`)

doc/source/whatsnew/v1.4.0.rst

@@ -1050,6 +1050,7 @@ Other
 - Bug in :meth:`Series.replace` raising ``ValueError`` when using ``regex=True`` with a :class:`Series` containing ``np.nan`` values (:issue:`43344`)
 - Bug in :meth:`DataFrame.to_records` where an incorrect ``n`` was used when missing names were replaced by ``level_n`` (:issue:`44818`)
 - Bug in :meth:`DataFrame.eval` where ``resolvers`` argument was overriding the default resolvers (:issue:`34966`)
+- :meth:`Series.__repr__` and :meth:`DataFrame.__repr__` no longer replace all null-values in indexes with "NaN" but use their real string-representations. "NaN" is used only for ``float("nan")`` (:issue:`45263`)
 
 .. ---------------------------------------------------------------------------
 

doc/source/whatsnew/v1.5.0.rst

@@ -185,7 +185,7 @@ Timezones
 
 Numeric
 ^^^^^^^
--
+- Bug in operations with array-likes with ``dtype="boolean"`` and :attr:`NA` incorrectly altering the array in-place (:issue:`45421`)
 -
 
 Conversion

pandas/_libs/missing.pyi

@@ -14,3 +14,4 @@ def checknull(val: object, inf_as_na: bool = ...) -> bool: ...
 def isnaobj(arr: np.ndarray, inf_as_na: bool = ...) -> npt.NDArray[np.bool_]: ...
 def isnaobj2d(arr: np.ndarray, inf_as_na: bool = ...) -> npt.NDArray[np.bool_]: ...
 def is_numeric_na(values: np.ndarray) -> npt.NDArray[np.bool_]: ...
+def is_float_nan(values: np.ndarray) -> npt.NDArray[np.bool_]: ...

pandas/_libs/missing.pyx

@@ -248,6 +248,31 @@ cdef bint checknull_with_nat_and_na(object obj):
     return checknull_with_nat(obj) or obj is C_NA
 
 
+@cython.wraparound(False)
+@cython.boundscheck(False)
+def is_float_nan(values: ndarray) -> ndarray:
+    """
+    True for elements which correspond to a float nan
+
+    Returns
+    -------
+    ndarray[bool]
+    """
+    cdef:
+        ndarray[uint8_t] result
+        Py_ssize_t i, N
+        object val
+
+    N = len(values)
+    result = np.zeros(N, dtype=np.uint8)
+
+    for i in range(N):
+        val = values[i]
+        if util.is_nan(val):
+            result[i] = True
+    return result.view(bool)
+
+
 @cython.wraparound(False)
 @cython.boundscheck(False)
 def is_numeric_na(values: ndarray) -> ndarray:

pandas/core/arrays/boolean.py

@@ -494,7 +494,8 @@ def _arith_method(self, other, op):
         if mask is None:
             mask = self._mask
             if other is libmissing.NA:
-                mask |= True
+                # GH#45421 don't alter inplace
+                mask = mask | True
         else:
             mask = self._mask | mask
 

pandas/core/arrays/datetimelike.py

@@ -430,14 +430,6 @@ def astype(self, dtype, copy: bool = True):
         elif is_integer_dtype(dtype):
             # we deliberately ignore int32 vs. int64 here.
             # See https://github.com/pandas-dev/pandas/issues/24381 for more.
-            warnings.warn(
-                f"casting {self.dtype} values to int64 with .astype(...) is "
-                "deprecated and will raise in a future version. "
-                "Use .view(...) instead.",
-                FutureWarning,
-                stacklevel=find_stack_level(),
-            )
-
             values = self.asi8
 
             if is_unsigned_integer_dtype(dtype):

pandas/core/dtypes/astype.py

@@ -112,13 +112,6 @@ def astype_nansafe(
 
     elif is_datetime64_dtype(arr.dtype):
         if dtype == np.int64:
-            warnings.warn(
-                f"casting {arr.dtype} values to int64 with .astype(...) "
-                "is deprecated and will raise in a future version. "
-                "Use .view(...) instead.",
-                FutureWarning,
-                stacklevel=find_stack_level(),
-            )
             if isna(arr).any():
                 raise ValueError("Cannot convert NaT values to integer")
             return arr.view(dtype)
@@ -131,13 +124,6 @@ def astype_nansafe(
 
     elif is_timedelta64_dtype(arr.dtype):
         if dtype == np.int64:
-            warnings.warn(
-                f"casting {arr.dtype} values to int64 with .astype(...) "
-                "is deprecated and will raise in a future version. "
-                "Use .view(...) instead.",
-                FutureWarning,
-                stacklevel=find_stack_level(),
-            )
             if isna(arr).any():
                 raise ValueError("Cannot convert NaT values to integer")
             return arr.view(dtype)

pandas/core/dtypes/cast.py

@@ -20,6 +20,7 @@
 )
 import warnings
 
+from dateutil.parser import ParserError
 import numpy as np
 
 from pandas._libs import lib
@@ -1336,9 +1337,8 @@ def maybe_cast_to_datetime(
                             value = dta.tz_localize("UTC").tz_convert(dtype.tz)
                 except OutOfBoundsDatetime:
                     raise
-                except ValueError:
-                    # TODO(GH#40048): only catch dateutil's ParserError
-                    #  once we can reliably import it in all supported versions
+                except ParserError:
+                    # Note: this is dateutil's ParserError, not ours.
                     pass
 
         elif getattr(vdtype, "kind", None) in ["m", "M"]:

pandas/core/frame.py

@@ -93,7 +93,6 @@
 )
 
 from pandas.core.dtypes.cast import (
-    can_hold_element,
     construct_1d_arraylike_from_scalar,
     construct_2d_arraylike_from_scalar,
     find_common_type,
@@ -3882,18 +3881,11 @@ def _set_value(
 
             series = self._get_item_cache(col)
             loc = self.index.get_loc(index)
-            dtype = series.dtype
-            if isinstance(dtype, np.dtype) and dtype.kind not in ["m", "M"]:
-                # otherwise we have EA values, and this check will be done
-                #  via setitem_inplace
-                if not can_hold_element(series._values, value):
-                    # We'll go through loc and end up casting.
-                    raise TypeError
-
-            series._mgr.setitem_inplace(loc, value)
-            # Note: trying to use series._set_value breaks tests in
-            #  tests.frame.indexing.test_indexing and tests.indexing.test_partial
-        except (KeyError, TypeError):
+
+            # series._set_value will do validation that may raise TypeError
+            #  or ValueError
+            series._set_value(loc, value, takeable=True)
+        except (KeyError, TypeError, ValueError):
             # set using a non-recursive method & reset the cache
             if takeable:
                 self.iloc[index, col] = value

pandas/core/generic.py

@@ -285,22 +285,6 @@ def _init_mgr(
                 mgr = mgr.astype(dtype=dtype)
         return mgr
 
-    @classmethod
-    def _from_mgr(cls, mgr: Manager):
-        """
-        Fastpath to create a new DataFrame/Series from just a BlockManager/ArrayManager.
-
-        Notes
-        -----
-        Skips setting `_flags` attribute; caller is responsible for doing so.
-        """
-        obj = cls.__new__(cls)
-        object.__setattr__(obj, "_is_copy", None)
-        object.__setattr__(obj, "_mgr", mgr)
-        object.__setattr__(obj, "_item_cache", {})
-        object.__setattr__(obj, "_attrs", {})
-        return obj
-
     def _as_manager(self: NDFrameT, typ: str, copy: bool_t = True) -> NDFrameT:
         """
         Private helper function to create a DataFrame with specific manager.
@@ -2068,6 +2052,41 @@ def empty(self) -> bool_t:
     def __array__(self, dtype: npt.DTypeLike | None = None) -> np.ndarray:
         return np.asarray(self._values, dtype=dtype)
 
+    def __array_wrap__(
+        self,
+        result: np.ndarray,
+        context: tuple[Callable, tuple[Any, ...], int] | None = None,
+    ):
+        """
+        Gets called after a ufunc and other functions.
+
+        Parameters
+        ----------
+        result: np.ndarray
+            The result of the ufunc or other function called on the NumPy array
+            returned by __array__
+        context: tuple of (func, tuple, int)
+            This parameter is returned by ufuncs as a 3-element tuple: (name of the
+            ufunc, arguments of the ufunc, domain of the ufunc), but is not set by
+            other numpy functions.q
+
+        Notes
+        -----
+        Series implements __array_ufunc_ so this not called for ufunc on Series.
+        """
+        # Note: at time of dask 2022.01.0, this is still used by dask
+        res = lib.item_from_zerodim(result)
+        if is_scalar(res):
+            # e.g. we get here with np.ptp(series)
+            # ptp also requires the item_from_zerodim
+            return res
+        d = self._construct_axes_dict(self._AXIS_ORDERS, copy=False)
+        # error: Argument 1 to "NDFrame" has incompatible type "ndarray";
+        # expected "BlockManager"
+        return self._constructor(res, **d).__finalize__(  # type: ignore[arg-type]
+            self, method="__array_wrap__"
+        )
+
     @final
     def __array_ufunc__(
         self, ufunc: np.ufunc, method: str, *inputs: Any, **kwargs: Any