Merge remote-tracking branch 'upstream/master' into move-metadata-to-cfg

Author: fangchenli

.pre-commit-config.yaml

@@ -168,6 +168,12 @@ repos:
             pandas/tests/io/excel/test_writers\.py
             |pandas/tests/io/pytables/common\.py
             |pandas/tests/io/pytables/test_store\.py$
+    -   id: no-pandas-api-types
+        name: Check code for instances of pd.api.types
+        entry: (pd|pandas)\.api\.types\.
+        language: pygrep
+        types: [python]
+        files: ^pandas/tests/
 -   repo: https://github.com/asottile/yesqa
     rev: v1.2.2
     hooks:

asv_bench/benchmarks/reshape.py

@@ -5,6 +5,7 @@
 
 import pandas as pd
 from pandas import DataFrame, MultiIndex, date_range, melt, wide_to_long
+from pandas.api.types import CategoricalDtype
 
 
 class Melt:
@@ -196,7 +197,7 @@ def setup(self):
         categories = list(string.ascii_letters[:12])
         s = pd.Series(
             np.random.choice(categories, size=1000000),
-            dtype=pd.api.types.CategoricalDtype(categories),
+            dtype=CategoricalDtype(categories),
         )
         self.s = s
 

doc/source/conf.py

@@ -164,7 +164,7 @@
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
-# language = None
+language = "en"
 
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:

doc/source/user_guide/timeseries.rst

@@ -2605,17 +2605,10 @@ For example, to localize and convert a naive stamp to time zone aware.
    s_naive.dt.tz_localize("UTC").dt.tz_convert("US/Eastern")
 
 Time zone information can also be manipulated using the ``astype`` method.
-This method can localize and convert time zone naive timestamps or
-convert time zone aware timestamps.
+This method can convert between different timezone-aware dtypes.
 
 .. ipython:: python
 
-   # localize and convert a naive time zone
-   s_naive.astype("datetime64[ns, US/Eastern]")
-
-   # make an aware tz naive
-   s_aware.astype("datetime64[ns]")
-
    # convert to a new time zone
    s_aware.astype("datetime64[ns, CET]")
 

doc/source/whatsnew/v1.2.0.rst

@@ -746,7 +746,7 @@ I/O
 - Parse missing values using :func:`read_json` with ``dtype=False`` to ``NaN`` instead of ``None`` (:issue:`28501`)
 - :meth:`read_fwf` was inferring compression with ``compression=None`` which was not consistent with the other ``read_*`` functions (:issue:`37909`)
 - :meth:`DataFrame.to_html` was ignoring ``formatters`` argument for ``ExtensionDtype`` columns (:issue:`36525`)
-- Bumped minimum xarray version to 0.12.3 to avoid reference to the removed ``Panel`` class (:issue:`27101`)
+- Bumped minimum xarray version to 0.12.3 to avoid reference to the removed ``Panel`` class (:issue:`27101`, :issue:`37983`)
 - :meth:`DataFrame.to_csv` was re-opening file-like handles that also implement ``os.PathLike`` (:issue:`38125`)
 - Bug in the conversion of a sliced ``pyarrow.Table`` with missing values to a DataFrame (:issue:`38525`)
 - Bug in :func:`read_sql_table` raising a ``sqlalchemy.exc.OperationalError`` when column names contained a percentage sign (:issue:`37517`)

doc/source/whatsnew/v1.3.0.rst

@@ -195,6 +195,7 @@ Deprecations
 - Deprecated :attr:`Rolling.win_type` returning ``"freq"`` (:issue:`38963`)
 - Deprecated :attr:`Rolling.is_datetimelike` (:issue:`38963`)
 - Deprecated :meth:`core.window.ewm.ExponentialMovingWindow.vol` (:issue:`39220`)
+- Using ``.astype`` to convert between ``datetime64[ns]`` dtype and :class:`DatetimeTZDtype` is deprecated and will raise in a future version, use ``obj.tz_localize`` or ``obj.dt.tz_localize`` instead (:issue:`38622`)
 -
 
 .. ---------------------------------------------------------------------------
@@ -235,6 +236,7 @@ Datetimelike
 - Bug in :meth:`DatetimeIndex.intersection`, :meth:`DatetimeIndex.symmetric_difference`, :meth:`PeriodIndex.intersection`, :meth:`PeriodIndex.symmetric_difference` always returning object-dtype when operating with :class:`CategoricalIndex` (:issue:`38741`)
 - Bug in :meth:`Series.where` incorrectly casting ``datetime64`` values to ``int64`` (:issue:`37682`)
 - Bug in :class:`Categorical` incorrectly typecasting ``datetime`` object to ``Timestamp`` (:issue:`38878`)
+- Bug in :meth:`Timestamp.round`, :meth:`Timestamp.floor`, :meth:`Timestamp.ceil` for values near the implementation bounds of :class:`Timestamp` (:issue:`39244`)
 - Bug in :func:`date_range` incorrectly creating :class:`DatetimeIndex` containing ``NaT`` instead of raising ``OutOfBoundsDatetime`` in corner cases (:issue:`24124`)
 
 Timedelta
@@ -274,14 +276,17 @@ Interval
 - Bug in :meth:`IntervalIndex.intersection` and :meth:`IntervalIndex.symmetric_difference` always returning object-dtype when operating with :class:`CategoricalIndex` (:issue:`38653`, :issue:`38741`)
 - Bug in :meth:`IntervalIndex.intersection` returning duplicates when at least one of both Indexes has duplicates which are present in the other (:issue:`38743`)
 - :meth:`IntervalIndex.union`, :meth:`IntervalIndex.intersection`, :meth:`IntervalIndex.difference`, and :meth:`IntervalIndex.symmetric_difference` now cast to the appropriate dtype instead of raising ``TypeError`` when operating with another :class:`IntervalIndex` with incompatible dtype (:issue:`39267`)
+- :meth:`PeriodIndex.union`, :meth:`PeriodIndex.intersection`, :meth:`PeriodIndex.symmetric_difference`, :meth:`PeriodIndex.difference` now cast to object dtype instead of raising ``IncompatibleFrequency`` when opearting with another :class:`PeriodIndex` with incompatible dtype (:issue:`??`)
 
 Indexing
 ^^^^^^^^
 - Bug in :meth:`CategoricalIndex.get_indexer` failing to raise ``InvalidIndexError`` when non-unique (:issue:`38372`)
 - Bug in inserting many new columns into a :class:`DataFrame` causing incorrect subsequent indexing behavior (:issue:`38380`)
+- Bug in :meth:`DataFrame.__setitem__` raising ``ValueError`` when setting multiple values to duplicate columns (:issue:`15695`)
 - Bug in :meth:`DataFrame.loc`, :meth:`Series.loc`, :meth:`DataFrame.__getitem__` and :meth:`Series.__getitem__` returning incorrect elements for non-monotonic :class:`DatetimeIndex` for string slices (:issue:`33146`)
 - Bug in :meth:`DataFrame.reindex` and :meth:`Series.reindex` with timezone aware indexes raising ``TypeError`` for ``method="ffill"`` and ``method="bfill"`` and specified ``tolerance`` (:issue:`38566`)
 - Bug in :meth:`DataFrame.__setitem__` raising ``ValueError`` with empty :class:`DataFrame` and specified columns for string indexer and non empty :class:`DataFrame` to set (:issue:`38831`)
+- Bug in :meth:`DataFrame.loc.__setitem__` raising ValueError when expanding unique column for :class:`DataFrame` with duplicate columns (:issue:`38521`)
 - Bug in :meth:`DataFrame.iloc.__setitem__` and :meth:`DataFrame.loc.__setitem__` with mixed dtypes when setting with a dictionary value (:issue:`38335`)
 - Bug in :meth:`DataFrame.loc` dropping levels of :class:`MultiIndex` when :class:`DataFrame` used as input has only one row (:issue:`10521`)
 - Bug in setting ``timedelta64`` values into numeric :class:`Series` failing to cast to object dtype (:issue:`39086`)

pandas/_libs/tslibs/period.pyx

@@ -1489,6 +1489,60 @@ cdef class PeriodMixin:
             return FR_SEC
         return base
 
+    @property
+    def start_time(self) -> Timestamp:
+        """
+        Get the Timestamp for the start of the period.
+
+        Returns
+        -------
+        Timestamp
+
+        See Also
+        --------
+        Period.end_time : Return the end Timestamp.
+        Period.dayofyear : Return the day of year.
+        Period.daysinmonth : Return the days in that month.
+        Period.dayofweek : Return the day of the week.
+
+        Examples
+        --------
+        >>> period = pd.Period('2012-1-1', freq='D')
+        >>> period
+        Period('2012-01-01', 'D')
+
+        >>> period.start_time
+        Timestamp('2012-01-01 00:00:00')
+
+        >>> period.end_time
+        Timestamp('2012-01-01 23:59:59.999999999')
+        """
+        return self.to_timestamp(how="start")
+
+    @property
+    def end_time(self) -> Timestamp:
+        return self.to_timestamp(how="end")
+
+    def _require_matching_freq(self, other, base=False):
+        # See also arrays.period.raise_on_incompatible
+        if is_offset_object(other):
+            other_freq = other
+        else:
+            other_freq = other.freq
+
+        if base:
+            condition = self.freq.base != other_freq.base
+        else:
+            condition = self.freq != other_freq
+
+        if condition:
+            msg = DIFFERENT_FREQ.format(
+                cls=type(self).__name__,
+                own_freq=self.freqstr,
+                other_freq=other_freq.freqstr,
+            )
+            raise IncompatibleFrequency(msg)
+
 
 cdef class _Period(PeriodMixin):
 
@@ -1551,10 +1605,7 @@ cdef class _Period(PeriodMixin):
                     return False
                 elif op == Py_NE:
                     return True
-                msg = DIFFERENT_FREQ.format(cls=type(self).__name__,
-                                            own_freq=self.freqstr,
-                                            other_freq=other.freqstr)
-                raise IncompatibleFrequency(msg)
+                self._require_matching_freq(other)
             return PyObject_RichCompareBool(self.ordinal, other.ordinal, op)
         elif other is NaT:
             return _nat_scalar_rules[op]
@@ -1563,15 +1614,15 @@ cdef class _Period(PeriodMixin):
     def __hash__(self):
         return hash((self.ordinal, self.freqstr))
 
-    def _add_delta(self, other) -> "Period":
+    def _add_timedeltalike_scalar(self, other) -> "Period":
         cdef:
-            int64_t nanos, offset_nanos
+            int64_t nanos, base_nanos
 
         if is_tick_object(self.freq):
             nanos = delta_to_nanoseconds(other)
-            offset_nanos = self.freq.base.nanos
-            if nanos % offset_nanos == 0:
-                ordinal = self.ordinal + (nanos // offset_nanos)
+            base_nanos = self.freq.base.nanos
+            if nanos % base_nanos == 0:
+                ordinal = self.ordinal + (nanos // base_nanos)
                 return Period(ordinal=ordinal, freq=self.freq)
         raise IncompatibleFrequency("Input cannot be converted to "
                                     f"Period(freq={self.freqstr})")
@@ -1581,14 +1632,10 @@ cdef class _Period(PeriodMixin):
         cdef:
             int64_t ordinal
 
-        if other.base == self.freq.base:
-            ordinal = self.ordinal + other.n
-            return Period(ordinal=ordinal, freq=self.freq)
+        self._require_matching_freq(other, base=True)
 
-        msg = DIFFERENT_FREQ.format(cls=type(self).__name__,
-                                    own_freq=self.freqstr,
-                                    other_freq=other.freqstr)
-        raise IncompatibleFrequency(msg)
+        ordinal = self.ordinal + other.n
+        return Period(ordinal=ordinal, freq=self.freq)
 
     def __add__(self, other):
         if not is_period_object(self):
@@ -1598,7 +1645,7 @@ cdef class _Period(PeriodMixin):
             return other.__add__(self)
 
         if is_any_td_scalar(other):
-            return self._add_delta(other)
+            return self._add_timedeltalike_scalar(other)
         elif is_offset_object(other):
             return self._add_offset(other)
         elif other is NaT:
@@ -1635,11 +1682,7 @@ cdef class _Period(PeriodMixin):
             ordinal = self.ordinal - other * self.freq.n
             return Period(ordinal=ordinal, freq=self.freq)
         elif is_period_object(other):
-            if other.freq != self.freq:
-                msg = DIFFERENT_FREQ.format(cls=type(self).__name__,
-                                            own_freq=self.freqstr,
-                                            other_freq=other.freqstr)
-                raise IncompatibleFrequency(msg)
+            self._require_matching_freq(other)
             # GH 23915 - mul by base freq since __add__ is agnostic of n
             return (self.ordinal - other.ordinal) * self.freq.base
         elif other is NaT:
@@ -1677,40 +1720,6 @@ cdef class _Period(PeriodMixin):
 
         return Period(ordinal=ordinal, freq=freq)
 
-    @property
-    def start_time(self) -> Timestamp:
-        """
-        Get the Timestamp for the start of the period.
-
-        Returns
-        -------
-        Timestamp
-
-        See Also
-        --------
-        Period.end_time : Return the end Timestamp.
-        Period.dayofyear : Return the day of year.
-        Period.daysinmonth : Return the days in that month.
-        Period.dayofweek : Return the day of the week.
-
-        Examples
-        --------
-        >>> period = pd.Period('2012-1-1', freq='D')
-        >>> period
-        Period('2012-01-01', 'D')
-
-        >>> period.start_time
-        Timestamp('2012-01-01 00:00:00')
-
-        >>> period.end_time
-        Timestamp('2012-01-01 23:59:59.999999999')
-        """
-        return self.to_timestamp(how='S')
-
-    @property
-    def end_time(self) -> Timestamp:
-        return self.to_timestamp(how="end")
-
     def to_timestamp(self, freq=None, how='start', tz=None) -> Timestamp:
         """
         Return the Timestamp representation of the Period.

pandas/_libs/tslibs/timestamps.pyx

@@ -8,6 +8,8 @@ shadows the python class, where we do any heavy lifting.
 """
 import warnings
 
+cimport cython
+
 import numpy as np
 
 cimport numpy as cnp
@@ -153,32 +155,69 @@ class RoundTo:
         return 4
 
 
-cdef inline _floor_int64(values, unit):
-    return values - np.remainder(values, unit)
+cdef inline ndarray[int64_t] _floor_int64(int64_t[:] values, int64_t unit):
+    cdef:
+        Py_ssize_t i, n = len(values)
+        ndarray[int64_t] result = np.empty(n, dtype="i8")
+        int64_t res, value
+
+    with cython.overflowcheck(True):
+        for i in range(n):
+            value = values[i]
+            if value == NPY_NAT:
+                res = NPY_NAT
+            else:
+                res = value - value % unit
+            result[i] = res
+
+    return result
+
+
+cdef inline ndarray[int64_t] _ceil_int64(int64_t[:] values, int64_t unit):
+    cdef:
+        Py_ssize_t i, n = len(values)
+        ndarray[int64_t] result = np.empty(n, dtype="i8")
+        int64_t res, value
 
-cdef inline _ceil_int64(values, unit):
-    return values + np.remainder(-values, unit)
+    with cython.overflowcheck(True):
+        for i in range(n):
+            value = values[i]
 
-cdef inline _rounddown_int64(values, unit):
+            if value == NPY_NAT:
+                res = NPY_NAT
+            else:
+                remainder = value % unit
+                if remainder == 0:
+                    res = value
+                else:
+                    res = value + (unit - remainder)
+
+            result[i] = res
+
+    return result
+
+
+cdef inline ndarray[int64_t] _rounddown_int64(values, int64_t unit):
     return _ceil_int64(values - unit//2, unit)
 
-cdef inline _roundup_int64(values, unit):
+
+cdef inline ndarray[int64_t] _roundup_int64(values, int64_t unit):
     return _floor_int64(values + unit//2, unit)
 
 
-def round_nsint64(values, mode, freq):
+def round_nsint64(values: np.ndarray, mode: RoundTo, freq) -> np.ndarray:
     """
     Applies rounding mode at given frequency
 
     Parameters
     ----------
-    values : :obj:`ndarray`
+    values : np.ndarray[int64_t]`
     mode : instance of `RoundTo` enumeration
     freq : str, obj
 
     Returns
     -------
-    :obj:`ndarray`
+    np.ndarray[int64_t]
     """
 
     unit = to_offset(freq).nanos

pandas/conftest.py

@@ -711,6 +711,24 @@ def float_frame():
     return DataFrame(tm.getSeriesData())
 
 
+@pytest.fixture
+def mixed_type_frame():
+    """
+    Fixture for DataFrame of float/int/string columns with RangeIndex
+    Columns are ['a', 'b', 'c', 'float32', 'int32'].
+    """
+    return DataFrame(
+        {
+            "a": 1.0,
+            "b": 2,
+            "c": "foo",
+            "float32": np.array([1.0] * 10, dtype="float32"),
+            "int32": np.array([1] * 10, dtype="int32"),
+        },
+        index=np.arange(10),
+    )
+
+
 # ----------------------------------------------------------------
 # Scalars
 # ----------------------------------------------------------------