Merge branch 'main' into bug_46673

Author: kapiliyer

doc/source/whatsnew/v1.4.4.rst

@@ -15,6 +15,7 @@ including other versions of pandas.
 Fixed regressions
 ~~~~~~~~~~~~~~~~~
 - Fixed regression in :func:`concat` materializing :class:`Index` during sorting even if :class:`Index` was already sorted (:issue:`47501`)
+- Fixed regression in setting ``None`` or non-string value into a ``string``-dtype Series using a mask (:issue:`47628`)
 -
 
 .. ---------------------------------------------------------------------------

doc/source/whatsnew/v1.5.0.rst

@@ -915,6 +915,7 @@ Missing
 ^^^^^^^
 - Bug in :meth:`Series.fillna` and :meth:`DataFrame.fillna` with ``downcast`` keyword not being respected in some cases where there are no NA values present (:issue:`45423`)
 - Bug in :meth:`Series.fillna` and :meth:`DataFrame.fillna` with :class:`IntervalDtype` and incompatible value raising instead of casting to a common (usually object) dtype (:issue:`45796`)
+- Bug in :meth:`Series.map` not respecting ``na_action`` argument if mapper is a ``dict`` or :class:`Series` (:issue:`47527`)
 - Bug in :meth:`DataFrame.interpolate` with object-dtype column not returning a copy with ``inplace=False`` (:issue:`45791`)
 - Bug in :meth:`DataFrame.dropna` allows to set both ``how`` and ``thresh`` incompatible arguments (:issue:`46575`)
 - Bug in :meth:`DataFrame.fillna` ignored ``axis`` when :class:`DataFrame` is single block (:issue:`47713`)
@@ -957,6 +958,7 @@ I/O
 - Bug in :func:`read_sas` that scrambled column names (:issue:`31243`)
 - Bug in :func:`read_sas` with RLE-compressed SAS7BDAT files that contain 0x00 control bytes (:issue:`47099`)
 - Bug in :func:`read_parquet` with ``use_nullable_dtypes=True`` where ``float64`` dtype was returned instead of nullable ``Float64`` dtype (:issue:`45694`)
+- Bug in :meth:`DataFrame.to_json` where ``PeriodDtype`` would not make the serialization roundtrip when read back with :meth:`read_json` (:issue:`44720`)
 
 Period
 ^^^^^^
@@ -1021,6 +1023,7 @@ Reshaping
 - Bug in :meth:`DataFrame.join` with a list when using suffixes to join DataFrames with duplicate column names (:issue:`46396`)
 - Bug in :meth:`DataFrame.pivot_table` with ``sort=False`` results in sorted index (:issue:`17041`)
 - Bug in :meth:`concat` when ``axis=1`` and ``sort=False`` where the resulting Index was a :class:`Int64Index` instead of a :class:`RangeIndex` (:issue:`46675`)
+- Bug in :meth:`wide_to_long` raises when ``stubnames`` is missing in columns and ``i`` contains string dtype column (:issue:`46044`)
 
 Sparse
 ^^^^^^

environment.yml

@@ -127,4 +127,4 @@ dependencies:
   # build the interactive terminal
   - jupyterlab >=3.4,<4
   - pip:
-      - jupyterlite==0.1.0b9
+      - jupyterlite==0.1.0b10

pandas/_libs/tslibs/dtypes.pxd

@@ -3,7 +3,7 @@ from numpy cimport int64_t
 from pandas._libs.tslibs.np_datetime cimport NPY_DATETIMEUNIT
 
 
-cdef str npy_unit_to_abbrev(NPY_DATETIMEUNIT unit)
+cpdef str npy_unit_to_abbrev(NPY_DATETIMEUNIT unit)
 cdef NPY_DATETIMEUNIT abbrev_to_npy_unit(str abbrev)
 cdef NPY_DATETIMEUNIT freq_group_code_to_npy_unit(int freq) nogil
 cpdef int64_t periods_per_day(NPY_DATETIMEUNIT reso=*) except? -1

pandas/_libs/tslibs/dtypes.pyi

@@ -8,6 +8,7 @@ _period_code_map: dict[str, int]
 def periods_per_day(reso: int) -> int: ...
 def periods_per_second(reso: int) -> int: ...
 def is_supported_unit(reso: int) -> bool: ...
+def npy_unit_to_abbrev(reso: int) -> str: ...
 
 class PeriodDtypeBase:
     _dtype_code: int  # PeriodDtypeCode

pandas/_libs/tslibs/dtypes.pyx

@@ -289,7 +289,7 @@ def is_supported_unit(NPY_DATETIMEUNIT reso):
     )
 
 
-cdef str npy_unit_to_abbrev(NPY_DATETIMEUNIT unit):
+cpdef str npy_unit_to_abbrev(NPY_DATETIMEUNIT unit):
     if unit == NPY_DATETIMEUNIT.NPY_FR_ns or unit == NPY_DATETIMEUNIT.NPY_FR_GENERIC:
         # generic -> default to nanoseconds
         return "ns"

pandas/_libs/tslibs/np_datetime.pxd

@@ -101,10 +101,18 @@ cpdef cnp.ndarray astype_overflowsafe(
     cnp.ndarray values,  # ndarray[datetime64[anyunit]]
     cnp.dtype dtype,  # ndarray[datetime64[anyunit]]
     bint copy=*,
+    bint round_ok=*,
 )
 cdef int64_t get_conversion_factor(NPY_DATETIMEUNIT from_unit, NPY_DATETIMEUNIT to_unit) except? -1
 
 cdef bint cmp_dtstructs(npy_datetimestruct* left, npy_datetimestruct* right, int op)
 cdef get_implementation_bounds(
     NPY_DATETIMEUNIT reso, npy_datetimestruct *lower, npy_datetimestruct *upper
 )
+
+cdef int64_t convert_reso(
+    int64_t value,
+    NPY_DATETIMEUNIT from_reso,
+    NPY_DATETIMEUNIT to_reso,
+    bint round_ok,
+) except? -1

pandas/_libs/tslibs/np_datetime.pyi

@@ -9,7 +9,10 @@ class OutOfBoundsTimedelta(ValueError): ...
 def py_get_unit_from_dtype(dtype: np.dtype): ...
 def py_td64_to_tdstruct(td64: int, unit: int) -> dict: ...
 def astype_overflowsafe(
-    arr: np.ndarray, dtype: np.dtype, copy: bool = ...
+    arr: np.ndarray,
+    dtype: np.dtype,
+    copy: bool = ...,
+    round_ok: bool = ...,
 ) -> np.ndarray: ...
 def is_unitless(dtype: np.dtype) -> bool: ...
 def compare_mismatched_resolutions(

pandas/_libs/tslibs/np_datetime.pyx

@@ -1,5 +1,6 @@
 cimport cython
 from cpython.datetime cimport (
+    PyDateTime_CheckExact,
     PyDateTime_DATE_GET_HOUR,
     PyDateTime_DATE_GET_MICROSECOND,
     PyDateTime_DATE_GET_MINUTE,
@@ -229,7 +230,13 @@ def py_td64_to_tdstruct(int64_t td64, NPY_DATETIMEUNIT unit):
 
 
 cdef inline void pydatetime_to_dtstruct(datetime dt, npy_datetimestruct *dts):
-    dts.year = PyDateTime_GET_YEAR(dt)
+    if PyDateTime_CheckExact(dt):
+        dts.year = PyDateTime_GET_YEAR(dt)
+    else:
+        # We use dt.year instead of PyDateTime_GET_YEAR because with Timestamp
+        #  we override year such that PyDateTime_GET_YEAR is incorrect.
+        dts.year = dt.year
+
     dts.month = PyDateTime_GET_MONTH(dt)
     dts.day = PyDateTime_GET_DAY(dt)
     dts.hour = PyDateTime_DATE_GET_HOUR(dt)
@@ -282,6 +289,7 @@ cpdef ndarray astype_overflowsafe(
     ndarray values,
     cnp.dtype dtype,
     bint copy=True,
+    bint round_ok=True,
 ):
     """
     Convert an ndarray with datetime64[X] to datetime64[Y]
@@ -314,20 +322,24 @@ cpdef ndarray astype_overflowsafe(
             "datetime64/timedelta64 values and dtype must have a unit specified"
         )
 
-    if (<object>values).dtype.byteorder == ">":
-        # GH#29684 we incorrectly get OutOfBoundsDatetime if we dont swap
-        values = values.astype(values.dtype.newbyteorder("<"))
-
     if from_unit == to_unit:
         # Check this before allocating result for perf, might save some memory
         if copy:
             return values.copy()
         return values
 
     elif from_unit > to_unit:
-        # e.g. ns -> us, so there is no risk of overflow, so we can use
-        #  numpy's astype safely. Note there _is_ risk of truncation.
-        return values.astype(dtype)
+        if round_ok:
+            # e.g. ns -> us, so there is no risk of overflow, so we can use
+            #  numpy's astype safely. Note there _is_ risk of truncation.
+            return values.astype(dtype)
+        else:
+            iresult2 = astype_round_check(values.view("i8"), from_unit, to_unit)
+            return iresult2.view(dtype)
+
+    if (<object>values).dtype.byteorder == ">":
+        # GH#29684 we incorrectly get OutOfBoundsDatetime if we dont swap
+        values = values.astype(values.dtype.newbyteorder("<"))
 
     cdef:
         ndarray i8values = values.view("i8")
@@ -356,10 +368,11 @@ cpdef ndarray astype_overflowsafe(
                 check_dts_bounds(&dts, to_unit)
             except OutOfBoundsDatetime as err:
                 if is_td:
-                    tdval = np.timedelta64(value).view(values.dtype)
+                    from_abbrev = np.datetime_data(values.dtype)[0]
+                    np_val = np.timedelta64(value, from_abbrev)
                     msg = (
-                        "Cannot convert {tdval} to {dtype} without overflow"
-                        .format(tdval=str(tdval), dtype=str(dtype))
+                        "Cannot convert {np_val} to {dtype} without overflow"
+                        .format(np_val=str(np_val), dtype=str(dtype))
                     )
                     raise OutOfBoundsTimedelta(msg) from err
                 else:
@@ -453,6 +466,52 @@ cdef int op_to_op_code(op):
         return Py_GT
 
 
+cdef ndarray astype_round_check(
+    ndarray i8values,
+    NPY_DATETIMEUNIT from_unit,
+    NPY_DATETIMEUNIT to_unit
+):
+    # cases with from_unit > to_unit, e.g. ns->us, raise if the conversion
+    #  involves truncation, e.g. 1500ns->1us
+    cdef:
+        Py_ssize_t i, N = i8values.size
+
+        # equiv: iresult = np.empty((<object>i8values).shape, dtype="i8")
+        ndarray iresult = cnp.PyArray_EMPTY(
+            i8values.ndim, i8values.shape, cnp.NPY_INT64, 0
+        )
+        cnp.broadcast mi = cnp.PyArray_MultiIterNew2(iresult, i8values)
+
+        # Note the arguments to_unit, from unit are swapped vs how they
+        #  are passed when going to a higher-frequency reso.
+        int64_t mult = get_conversion_factor(to_unit, from_unit)
+        int64_t value, mod
+
+    for i in range(N):
+        # Analogous to: item = i8values[i]
+        value = (<int64_t*>cnp.PyArray_MultiIter_DATA(mi, 1))[0]
+
+        if value == NPY_DATETIME_NAT:
+            new_value = NPY_DATETIME_NAT
+        else:
+            new_value, mod = divmod(value, mult)
+            if mod != 0:
+                # TODO: avoid runtime import
+                from pandas._libs.tslibs.dtypes import npy_unit_to_abbrev
+                from_abbrev = npy_unit_to_abbrev(from_unit)
+                to_abbrev = npy_unit_to_abbrev(to_unit)
+                raise ValueError(
+                    f"Cannot losslessly cast '{value} {from_abbrev}' to {to_abbrev}"
+                )
+
+        # Analogous to: iresult[i] = new_value
+        (<int64_t*>cnp.PyArray_MultiIter_DATA(mi, 0))[0] = new_value
+
+        cnp.PyArray_MultiIter_NEXT(mi)
+
+    return iresult
+
+
 @cython.overflowcheck(True)
 cdef int64_t get_conversion_factor(NPY_DATETIMEUNIT from_unit, NPY_DATETIMEUNIT to_unit) except? -1:
     """
@@ -489,5 +548,59 @@ cdef int64_t get_conversion_factor(NPY_DATETIMEUNIT from_unit, NPY_DATETIMEUNIT
         return 1000 * get_conversion_factor(NPY_DATETIMEUNIT.NPY_FR_fs, to_unit)
     elif from_unit == NPY_DATETIMEUNIT.NPY_FR_fs:
         return 1000 * get_conversion_factor(NPY_DATETIMEUNIT.NPY_FR_as, to_unit)
+
+
+cdef int64_t convert_reso(
+    int64_t value,
+    NPY_DATETIMEUNIT from_reso,
+    NPY_DATETIMEUNIT to_reso,
+    bint round_ok,
+) except? -1:
+    cdef:
+        int64_t res_value, mult, div, mod
+
+    if from_reso == to_reso:
+        return value
+
+    elif to_reso < from_reso:
+        # e.g. ns -> us, no risk of overflow, but can be lossy rounding
+        mult = get_conversion_factor(to_reso, from_reso)
+        div, mod = divmod(value, mult)
+        if mod > 0 and not round_ok:
+            raise ValueError("Cannot losslessly convert units")
+
+        # Note that when mod > 0, we follow np.timedelta64 in always
+        #  rounding down.
+        res_value = div
+
+    elif (
+        from_reso == NPY_FR_Y
+        or from_reso == NPY_FR_M
+        or to_reso == NPY_FR_Y
+        or to_reso == NPY_FR_M
+    ):
+        # Converting by multiplying isn't _quite_ right bc the number of
+        #  seconds in a month/year isn't fixed.
+        res_value = _convert_reso_with_dtstruct(value, from_reso, to_reso)
+
     else:
-        raise ValueError(from_unit, to_unit)
+        # e.g. ns -> us, risk of overflow, but no risk of lossy rounding
+        mult = get_conversion_factor(from_reso, to_reso)
+        with cython.overflowcheck(True):
+            # Note: caller is responsible for re-raising as OutOfBoundsTimedelta
+            res_value = value * mult
+
+    return res_value
+
+
+cdef int64_t _convert_reso_with_dtstruct(
+    int64_t value,
+    NPY_DATETIMEUNIT from_unit,
+    NPY_DATETIMEUNIT to_unit,
+) except? -1:
+    cdef:
+        npy_datetimestruct dts
+
+    pandas_datetime_to_datetimestruct(value, from_unit, &dts)
+    check_dts_bounds(&dts, to_unit)
+    return npy_datetimestruct_to_datetime(to_unit, &dts)

pandas/_libs/tslibs/offsets.pyi

@@ -111,6 +111,8 @@ def to_offset(freq: timedelta | str) -> BaseOffset: ...
 
 class Tick(SingleConstructorOffset):
     _reso: int
+    _prefix: str
+    _td64_unit: str
     def __init__(self, n: int = ..., normalize: bool = ...) -> None: ...
     @property
     def delta(self) -> Timedelta: ...