Merge remote-tracking branch 'upstream/main' into non-nano-strftime

Author: MarcoGorelli

.github/workflows/ubuntu.yml

@@ -77,7 +77,7 @@ jobs:
           - name: "Numpy Dev"
             env_file: actions-310-numpydev.yaml
             pattern: "not slow and not network and not single_cpu"
-            test_args: "-W error::DeprecationWarning:numpy -W error::FutureWarning:numpy"
+            test_args: "-W error::DeprecationWarning -W error::FutureWarning"
             error_on_warnings: "0"
         exclude:
           - env_file: actions-38.yaml

doc/source/whatsnew/v1.5.1.rst

@@ -76,14 +76,14 @@ Fixed regressions
 - Fixed regression in :meth:`DataFrame.loc` raising ``FutureWarning`` when setting an empty :class:`DataFrame` (:issue:`48480`)
 - Fixed regression in :meth:`DataFrame.describe` raising ``TypeError`` when result contains ``NA`` (:issue:`48778`)
 - Fixed regression in :meth:`DataFrame.plot` ignoring invalid ``colormap`` for ``kind="scatter"`` (:issue:`48726`)
-- Fixed regression in :meth:`MultiIndex.values`` resetting ``freq`` attribute of underlying :class:`Index` object (:issue:`49054`)
+- Fixed regression in :meth:`MultiIndex.values` resetting ``freq`` attribute of underlying :class:`Index` object (:issue:`49054`)
 - Fixed performance regression in :func:`factorize` when ``na_sentinel`` is not ``None`` and ``sort=False`` (:issue:`48620`)
 - Fixed regression causing an ``AttributeError`` during warning emitted if the provided table name in :meth:`DataFrame.to_sql` and the table name actually used in the database do not match (:issue:`48733`)
 - Fixed regression in :func:`to_datetime` when ``arg`` was a date string with nanosecond and ``format`` contained ``%f`` would raise a ``ValueError`` (:issue:`48767`)
-- Fixed regression in :func:`assert_frame_equal` raising for :class:`MultiIndex` with :class:`Categorical` and ``check_like=True`` (:issue:`48975`)
+- Fixed regression in :func:`testing.assert_frame_equal` raising for :class:`MultiIndex` with :class:`Categorical` and ``check_like=True`` (:issue:`48975`)
 - Fixed regression in :meth:`DataFrame.fillna` replacing wrong values for ``datetime64[ns]`` dtype and ``inplace=True`` (:issue:`48863`)
 - Fixed :meth:`.DataFrameGroupBy.size` not returning a Series when ``axis=1`` (:issue:`48738`)
-- Fixed Regression in :meth:`DataFrameGroupBy.apply` when user defined function is called on an empty dataframe (:issue:`47985`)
+- Fixed Regression in :meth:`.DataFrameGroupBy.apply` when user defined function is called on an empty dataframe (:issue:`47985`)
 - Fixed regression in :meth:`DataFrame.apply` when passing non-zero ``axis`` via keyword argument (:issue:`48656`)
 - Fixed regression in :meth:`Series.groupby` and :meth:`DataFrame.groupby` when the grouper is a nullable data type (e.g. :class:`Int64`) or a PyArrow-backed string array, contains null values, and ``dropna=False`` (:issue:`48794`)
 - Fixed performance regression in :meth:`Series.isin` with mismatching dtypes (:issue:`49162`)
@@ -99,7 +99,7 @@ Bug fixes
 ~~~~~~~~~
 - Bug in :meth:`Series.__getitem__` not falling back to positional for integer keys and boolean :class:`Index` (:issue:`48653`)
 - Bug in :meth:`DataFrame.to_hdf` raising ``AssertionError`` with boolean index (:issue:`48667`)
-- Bug in :func:`assert_index_equal` for extension arrays with non matching ``NA`` raising ``ValueError`` (:issue:`48608`)
+- Bug in :func:`testing.assert_index_equal` for extension arrays with non matching ``NA`` raising ``ValueError`` (:issue:`48608`)
 - Bug in :meth:`DataFrame.pivot_table` raising unexpected ``FutureWarning`` when setting datetime column as index (:issue:`48683`)
 - Bug in :meth:`DataFrame.sort_values` emitting unnecessary ``FutureWarning`` when called on :class:`DataFrame` with boolean sparse columns (:issue:`48784`)
 - Bug in :class:`.arrays.ArrowExtensionArray` with a comparison operator to an invalid object would not raise a ``NotImplementedError`` (:issue:`48833`)

doc/source/whatsnew/v1.5.3.rst

@@ -1,7 +1,7 @@
 .. _whatsnew_153:
 
-What's new in 1.5.3 (December ??, 2022)
----------------------------------------
+What's new in 1.5.3 (January 18, 2023)
+--------------------------------------
 
 These are the changes in pandas 1.5.3. See :ref:`release` for a full changelog
 including other versions of pandas.
@@ -15,12 +15,11 @@ Fixed regressions
 ~~~~~~~~~~~~~~~~~
 - Fixed performance regression in :meth:`Series.isin` when ``values`` is empty (:issue:`49839`)
 - Fixed regression in :meth:`DataFrame.memory_usage` showing unnecessary ``FutureWarning`` when :class:`DataFrame` is empty (:issue:`50066`)
-- Fixed regression in :meth:`DataFrameGroupBy.transform` when used with ``as_index=False`` (:issue:`49834`)
+- Fixed regression in :meth:`.DataFrameGroupBy.transform` when used with ``as_index=False`` (:issue:`49834`)
 - Enforced reversion of ``color`` as an alias for ``c`` and ``size`` as an alias for ``s`` in function :meth:`DataFrame.plot.scatter` (:issue:`49732`)
-- Fixed regression in :meth:`SeriesGroupBy.apply` setting a ``name`` attribute on the result if the result was a :class:`DataFrame` (:issue:`49907`)
+- Fixed regression in :meth:`.SeriesGroupBy.apply` setting a ``name`` attribute on the result if the result was a :class:`DataFrame` (:issue:`49907`)
 - Fixed performance regression in setting with the :meth:`~DataFrame.at` indexer (:issue:`49771`)
 - Fixed regression in :func:`to_datetime` raising ``ValueError`` when parsing array of ``float`` containing ``np.nan`` (:issue:`50237`)
--
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_153.bug_fixes:
@@ -48,7 +47,6 @@ Other
     as pandas works toward compatibility with SQLAlchemy 2.0.
 
 - Reverted deprecation (:issue:`45324`) of behavior of :meth:`Series.__getitem__` and :meth:`Series.__setitem__` slicing with an integer :class:`Index`; this will remain positional (:issue:`49612`)
--
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_153.contributors:

doc/source/whatsnew/v2.0.0.rst

@@ -41,6 +41,7 @@ The ``use_nullable_dtypes`` keyword argument has been expanded to the following
 * :func:`read_excel`
 * :func:`read_html`
 * :func:`read_xml`
+* :func:`read_json`
 * :func:`read_sql`
 * :func:`read_sql_query`
 * :func:`read_sql_table`
@@ -56,6 +57,7 @@ to select the nullable dtypes implementation.
 * :func:`read_excel`
 * :func:`read_html`
 * :func:`read_xml`
+* :func:`read_json`
 * :func:`read_parquet`
 * :func:`read_orc`
 * :func:`read_feather`
@@ -92,10 +94,9 @@ Copy-on-Write improvements
   was added to the following methods:
 
   - :meth:`DataFrame.reset_index` / :meth:`Series.reset_index`
-  - :meth:`DataFrame.set_index` / :meth:`Series.set_index`
+  - :meth:`DataFrame.set_index`
   - :meth:`DataFrame.set_axis` / :meth:`Series.set_axis`
   - :meth:`DataFrame.rename_axis` / :meth:`Series.rename_axis`
-  - :meth:`DataFrame.rename_columns`
   - :meth:`DataFrame.reindex` / :meth:`Series.reindex`
   - :meth:`DataFrame.reindex_like` / :meth:`Series.reindex_like`
   - :meth:`DataFrame.assign`
@@ -175,8 +176,8 @@ These are bug fixes that might have notable behavior changes.
 
 .. _whatsnew_200.notable_bug_fixes.cumsum_cumprod_overflow:
 
-:meth:`.GroupBy.cumsum` and :meth:`.GroupBy.cumprod` overflow instead of lossy casting to float
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+:meth:`.DataFrameGroupBy.cumsum` and :meth:`.DataFrameGroupBy.cumprod` overflow instead of lossy casting to float
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 In previous versions we cast to float when applying ``cumsum`` and ``cumprod`` which
 lead to incorrect results even if the result could be hold by ``int64`` dtype.
@@ -822,7 +823,7 @@ Removal of prior version deprecations/changes
 
 Performance improvements
 ~~~~~~~~~~~~~~~~~~~~~~~~
-- Performance improvement in :meth:`.DataFrameGroupBy.median` and :meth:`.SeriesGroupBy.median` and :meth:`.GroupBy.cumprod` for nullable dtypes (:issue:`37493`)
+- Performance improvement in :meth:`.DataFrameGroupBy.median` and :meth:`.SeriesGroupBy.median` and :meth:`.DataFrameGroupBy.cumprod` for nullable dtypes (:issue:`37493`)
 - Performance improvement in :meth:`.DataFrameGroupBy.all`, :meth:`.DataFrameGroupBy.any`, :meth:`.SeriesGroupBy.all`, and :meth:`.SeriesGroupBy.any` for object dtype (:issue:`50623`)
 - Performance improvement in :meth:`MultiIndex.argsort` and :meth:`MultiIndex.sort_values` (:issue:`48406`)
 - Performance improvement in :meth:`MultiIndex.size` (:issue:`48723`)
@@ -952,6 +953,7 @@ Conversion
 Strings
 ^^^^^^^
 - Bug in :func:`pandas.api.dtypes.is_string_dtype` that would not return ``True`` for :class:`StringDtype` (:issue:`15585`)
+- Bug in converting string dtypes to "datetime64[ns]" or "timedelta64[ns]" incorrectly raising ``TypeError`` (:issue:`36153`)
 -
 
 Interval
@@ -1018,6 +1020,7 @@ I/O
 - Bug in :meth:`DataFrame.to_string` ignoring float formatter for extension arrays (:issue:`39336`)
 - Fixed memory leak which stemmed from the initialization of the internal JSON module (:issue:`49222`)
 - Fixed issue where :func:`json_normalize` would incorrectly remove leading characters from column names that matched the ``sep`` argument (:issue:`49861`)
+- Bug in :meth:`DataFrame.to_dict` not converting ``NA`` to ``None`` (:issue:`50795`)
 - Bug in :meth:`DataFrame.to_json` where it would segfault when failing to encode a string (:issue:`50307`)
 
 Period

pandas/compat/pickle_compat.py

@@ -120,7 +120,20 @@ def load_reduce(self):
     ),
     ("pandas.indexes.numeric", "Float64Index"): (
         "pandas.core.indexes.numeric",
-        "Float64Index",
+        "Index",  # updated in 50775
+    ),
+    # 50775, remove Int64Index, UInt64Index & Float64Index from codabase
+    ("pandas.core.indexes.numeric", "Int64Index"): (
+        "pandas.core.indexes.base",
+        "Index",
+    ),
+    ("pandas.core.indexes.numeric", "UInt64Index"): (
+        "pandas.core.indexes.base",
+        "Index",
+    ),
+    ("pandas.core.indexes.numeric", "Float64Index"): (
+        "pandas.core.indexes.base",
+        "Index",
     ),
 }
 

pandas/core/arrays/base.py

@@ -55,9 +55,11 @@
 
 from pandas.core.dtypes.cast import maybe_cast_to_extension_array
 from pandas.core.dtypes.common import (
+    is_datetime64_dtype,
     is_dtype_equal,
     is_list_like,
     is_scalar,
+    is_timedelta64_dtype,
     pandas_dtype,
 )
 from pandas.core.dtypes.dtypes import ExtensionDtype
@@ -580,6 +582,16 @@ def astype(self, dtype: AstypeArg, copy: bool = True) -> ArrayLike:
             cls = dtype.construct_array_type()
             return cls._from_sequence(self, dtype=dtype, copy=copy)
 
+        elif is_datetime64_dtype(dtype):
+            from pandas.core.arrays import DatetimeArray
+
+            return DatetimeArray._from_sequence(self, dtype=dtype, copy=copy)
+
+        elif is_timedelta64_dtype(dtype):
+            from pandas.core.arrays import TimedeltaArray
+
+            return TimedeltaArray._from_sequence(self, dtype=dtype, copy=copy)
+
         return np.array(self, dtype=dtype, copy=copy)
 
     def isna(self) -> np.ndarray | ExtensionArraySupportsAnyAll:

pandas/core/dtypes/cast.py

@@ -20,6 +20,10 @@
 import numpy as np
 
 from pandas._libs import lib
+from pandas._libs.missing import (
+    NA,
+    NAType,
+)
 from pandas._libs.tslibs import (
     NaT,
     OutOfBoundsDatetime,
@@ -176,7 +180,7 @@ def maybe_box_datetimelike(value: Scalar, dtype: Dtype | None = None) -> Scalar:
     return value
 
 
-def maybe_box_native(value: Scalar) -> Scalar:
+def maybe_box_native(value: Scalar | None | NAType) -> Scalar | None | NAType:
     """
     If passed a scalar cast the scalar to a python native type.
 
@@ -202,6 +206,8 @@ def maybe_box_native(value: Scalar) -> Scalar:
         value = bool(value)
     elif isinstance(value, (np.datetime64, np.timedelta64)):
         value = maybe_box_datetimelike(value)
+    elif value is NA:
+        value = None
     return value
 
 
@@ -1590,7 +1596,15 @@ def maybe_cast_to_integer_array(arr: list | np.ndarray, dtype: np.dtype) -> np.n
 
     try:
         if not isinstance(arr, np.ndarray):
-            casted = np.array(arr, dtype=dtype, copy=False)
+            with warnings.catch_warnings():
+                # We already disallow dtype=uint w/ negative numbers
+                # (test_constructor_coercion_signed_to_unsigned) so safe to ignore.
+                warnings.filterwarnings(
+                    "ignore",
+                    "NumPy will stop allowing conversion of out-of-bound Python int",
+                    DeprecationWarning,
+                )
+                casted = np.array(arr, dtype=dtype, copy=False)
         else:
             casted = arr.astype(dtype, copy=False)
     except OverflowError as err:

pandas/core/dtypes/generic.py

@@ -29,11 +29,6 @@
         TimedeltaArray,
     )
     from pandas.core.generic import NDFrame
-    from pandas.core.indexes.api import (
-        Float64Index,
-        Int64Index,
-        UInt64Index,
-    )
 
 
 # define abstract base classes to enable isinstance type checking on our
@@ -62,22 +57,10 @@ def _subclasscheck(cls, inst) -> bool:
     return meta(name, (), dct)
 
 
-ABCInt64Index = cast(
-    "Type[Int64Index]",
-    create_pandas_abc_type("ABCInt64Index", "_typ", ("int64index",)),
-)
-ABCUInt64Index = cast(
-    "Type[UInt64Index]",
-    create_pandas_abc_type("ABCUInt64Index", "_typ", ("uint64index",)),
-)
 ABCRangeIndex = cast(
     "Type[RangeIndex]",
     create_pandas_abc_type("ABCRangeIndex", "_typ", ("rangeindex",)),
 )
-ABCFloat64Index = cast(
-    "Type[Float64Index]",
-    create_pandas_abc_type("ABCFloat64Index", "_typ", ("float64index",)),
-)
 ABCMultiIndex = cast(
     "Type[MultiIndex]",
     create_pandas_abc_type("ABCMultiIndex", "_typ", ("multiindex",)),
@@ -109,10 +92,7 @@ def _subclasscheck(cls, inst) -> bool:
         "_typ",
         {
             "index",
-            "int64index",
             "rangeindex",
-            "float64index",
-            "uint64index",
             "numericindex",
             "multiindex",
             "datetimeindex",

pandas/core/methods/to_dict.py

@@ -6,7 +6,10 @@
 from pandas.util._exceptions import find_stack_level
 
 from pandas.core.dtypes.cast import maybe_box_native
-from pandas.core.dtypes.common import is_object_dtype
+from pandas.core.dtypes.common import (
+    is_extension_array_dtype,
+    is_object_dtype,
+)
 
 from pandas import DataFrame
 from pandas.core import common as com
@@ -88,16 +91,18 @@ def to_dict(
         # GH46470 Return quickly if orient series to avoid creating dtype objects
         return into_c((k, v) for k, v in df.items())
 
-    object_dtype_indices = [
-        i for i, col_dtype in enumerate(df.dtypes.values) if is_object_dtype(col_dtype)
+    box_native_indices = [
+        i
+        for i, col_dtype in enumerate(df.dtypes.values)
+        if is_object_dtype(col_dtype) or is_extension_array_dtype(col_dtype)
     ]
-    are_all_object_dtype_cols = len(object_dtype_indices) == len(df.dtypes)
+    are_all_object_dtype_cols = len(box_native_indices) == len(df.dtypes)
 
     if orient == "dict":
         return into_c((k, v.to_dict(into)) for k, v in df.items())
 
     elif orient == "list":
-        object_dtype_indices_as_set = set(object_dtype_indices)
+        object_dtype_indices_as_set = set(box_native_indices)
         return into_c(
             (
                 k,
@@ -110,7 +115,7 @@ def to_dict(
 
     elif orient == "split":
         data = df._create_data_for_split_and_tight_to_dict(
-            are_all_object_dtype_cols, object_dtype_indices
+            are_all_object_dtype_cols, box_native_indices
         )
 
         return into_c(
@@ -123,7 +128,7 @@ def to_dict(
 
     elif orient == "tight":
         data = df._create_data_for_split_and_tight_to_dict(
-            are_all_object_dtype_cols, object_dtype_indices
+            are_all_object_dtype_cols, box_native_indices
         )
 
         return into_c(
@@ -155,8 +160,8 @@ def to_dict(
             data = [
                 into_c(zip(columns, t)) for t in df.itertuples(index=False, name=None)
             ]
-            if object_dtype_indices:
-                object_dtype_indices_as_set = set(object_dtype_indices)
+            if box_native_indices:
+                object_dtype_indices_as_set = set(box_native_indices)
                 object_dtype_cols = {
                     col
                     for i, col in enumerate(df.columns)
@@ -176,8 +181,8 @@ def to_dict(
                 (t[0], dict(zip(df.columns, map(maybe_box_native, t[1:]))))
                 for t in df.itertuples(name=None)
             )
-        elif object_dtype_indices:
-            object_dtype_indices_as_set = set(object_dtype_indices)
+        elif box_native_indices:
+            object_dtype_indices_as_set = set(box_native_indices)
             is_object_dtype_by_index = [
                 i in object_dtype_indices_as_set for i in range(len(df.columns))
             ]

pandas/core/series.py

@@ -89,6 +89,7 @@
 from pandas.core.dtypes.common import (
     ensure_platform_int,
     is_dict_like,
+    is_extension_array_dtype,
     is_integer,
     is_iterator,
     is_list_like,
@@ -1832,7 +1833,7 @@ def to_dict(self, into: type[dict] = dict) -> dict:
         # GH16122
         into_c = com.standardize_mapping(into)
 
-        if is_object_dtype(self):
+        if is_object_dtype(self) or is_extension_array_dtype(self):
             return into_c((k, maybe_box_native(v)) for k, v in self.items())
         else:
             # Not an object dtype => all types will be the same so let the default