From 2c4c82224f1daeaa1e069f191fa3a23a6803264d Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 7 Jul 2020 13:19:42 -0500 Subject: [PATCH 1/7] Fixed apply_index Closes https://github.com/pandas-dev/pandas/issues/34580 --- doc/source/whatsnew/v1.1.0.rst | 1 + pandas/_libs/tslibs/offsets.pyx | 82 ++++++++++++++++++-- pandas/core/arrays/datetimes.py | 2 +- pandas/tests/tseries/offsets/test_offsets.py | 7 +- 4 files changed, 85 insertions(+), 7 deletions(-) diff --git a/doc/source/whatsnew/v1.1.0.rst b/doc/source/whatsnew/v1.1.0.rst index cee41f248fc60..ecdbfbbccb968 100644 --- a/doc/source/whatsnew/v1.1.0.rst +++ b/doc/source/whatsnew/v1.1.0.rst @@ -811,6 +811,7 @@ Deprecations - :meth:`DatetimeIndex.week` and `DatetimeIndex.weekofyear` are deprecated and will be removed in a future version, use :meth:`DatetimeIndex.isocalendar().week` instead (:issue:`33595`) - :meth:`DatetimeArray.week` and `DatetimeArray.weekofyear` are deprecated and will be removed in a future version, use :meth:`DatetimeArray.isocalendar().week` instead (:issue:`33595`) - :meth:`DateOffset.__call__` is deprecated and will be removed in a future version, use ``offset + other`` instead (:issue:`34171`) +- :meth:`~BusinessDay.apply_index` is deprecated and will be removed in a future version. Use ``offset + other`` instead (:issue:`34580`) - :meth:`DataFrame.tshift` and :meth:`Series.tshift` are deprecated and will be removed in a future version, use :meth:`DataFrame.shift` and :meth:`Series.shift` instead (:issue:`11631`) - Indexing an :class:`Index` object with a float key is deprecated, and will raise an ``IndexError`` in the future. You can manually convert to an integer key diff --git a/pandas/_libs/tslibs/offsets.pyx b/pandas/_libs/tslibs/offsets.pyx index e4d05e0d70e2f..113c9ce01c3ed 100644 --- a/pandas/_libs/tslibs/offsets.pyx +++ b/pandas/_libs/tslibs/offsets.pyx @@ -82,17 +82,45 @@ cdef bint _is_normalized(datetime dt): return True +def apply_wrapper_core(func, self, other): + result = func(self, other) + result = np.asarray(result) + + if self.normalize: + result = normalize_i8_timestamps(result.view("i8"), None) + + return result + + def apply_index_wraps(func): # Note: normally we would use `@functools.wraps(func)`, but this does # not play nicely with cython class methods - def wrapper(self, other) -> np.ndarray: + def wrapper(self, other): # other is a DatetimeArray + result = apply_wrapper_core(func, self, other) + result = type(other)(result) + warnings.warn("'Offset.apply_index(other)' is deprecated. " + "Use 'offset + other' instead.", FutureWarning) + return result - result = func(self, other) - result = np.asarray(result) + # do @functools.wraps(func) manually since it doesn't work on cdef funcs + wrapper.__name__ = func.__name__ + wrapper.__doc__ = func.__doc__ + try: + wrapper.__module__ = func.__module__ + except AttributeError: + # AttributeError: 'method_descriptor' object has no + # attribute '__module__' + pass + return wrapper - if self.normalize: - result = normalize_i8_timestamps(result.view("i8"), None) + +def apply_array_wraps(func): + # Note: normally we would use `@functools.wraps(func)`, but this does + # not play nicely with cython class methods + def wrapper(self, other) -> np.ndarray: + # other is a DatetimeArray + result = apply_wrapper_core(func, self, other) return result # do @functools.wraps(func) manually since it doesn't work on cdef funcs @@ -554,6 +582,10 @@ cdef class BaseOffset: raises NotImplementedError for offsets without a vectorized implementation. + .. deprecated:: 1.1.0 + + Use ``offset + dtindex`` instead. + Parameters ---------- index : DatetimeIndex @@ -567,6 +599,13 @@ cdef class BaseOffset: "does not have a vectorized implementation" ) + @apply_array_wraps + def _apply_array(self, dtindex): + raise NotImplementedError( + f"DateOffset subclass {type(self).__name__} " + "does not have a vectorized implementation" + ) + def rollback(self, dt) -> datetime: """ Roll provided date backward to next offset only if not on offset. @@ -1031,6 +1070,10 @@ cdef class RelativeDeltaOffset(BaseOffset): ------- ndarray[datetime64[ns]] """ + return self._apply_array(dtindex) + + @apply_array_wraps + def _apply_array(self, dtindex): dt64other = np.asarray(dtindex) kwds = self.kwds relativedelta_fast = { @@ -1360,6 +1403,10 @@ cdef class BusinessDay(BusinessMixin): @apply_index_wraps def apply_index(self, dtindex): + return self._apply_array(dtindex) + + @apply_array_wraps + def _apply_array(self, dtindex): i8other = dtindex.view("i8") return shift_bdays(i8other, self.n) @@ -1843,6 +1890,10 @@ cdef class YearOffset(SingleConstructorOffset): @apply_index_wraps def apply_index(self, dtindex): + return self._apply_array(dtindex) + + @apply_array_wraps + def _apply_array(self, dtindex): shifted = shift_quarters( dtindex.view("i8"), self.n, self.month, self._day_opt, modby=12 ) @@ -1996,6 +2047,10 @@ cdef class QuarterOffset(SingleConstructorOffset): @apply_index_wraps def apply_index(self, dtindex): + return self._apply_array(dtindex) + + @apply_array_wraps + def _apply_array(self, dtindex): shifted = shift_quarters( dtindex.view("i8"), self.n, self.startingMonth, self._day_opt ) @@ -2111,6 +2166,10 @@ cdef class MonthOffset(SingleConstructorOffset): @apply_index_wraps def apply_index(self, dtindex): + return self._apply_array(dtindex) + + @apply_array_wraps + def _apply_array(self, dtindex): shifted = shift_months(dtindex.view("i8"), self.n, self._day_opt) return shifted @@ -2248,6 +2307,12 @@ cdef class SemiMonthOffset(SingleConstructorOffset): @cython.wraparound(False) @cython.boundscheck(False) def apply_index(self, dtindex): + return self._apply_array(dtindex) + + @apply_array_wraps + @cython.wraparound(False) + @cython.boundscheck(False) + def _apply_array(self, dtindex): cdef: int64_t[:] i8other = dtindex.view("i8") Py_ssize_t i, count = len(i8other) @@ -2407,6 +2472,10 @@ cdef class Week(SingleConstructorOffset): @apply_index_wraps def apply_index(self, dtindex): + return self._apply_array(dtindex) + + @apply_array_wraps + def _apply_array(self, dtindex): if self.weekday is None: td = timedelta(days=7 * self.n) td64 = np.timedelta64(td, "ns") @@ -3185,6 +3254,9 @@ cdef class CustomBusinessDay(BusinessDay): def apply_index(self, dtindex): raise NotImplementedError + def _apply_array(self, dtindex): + raise NotImplementedError + def is_on_offset(self, dt: datetime) -> bool: if self.normalize and not _is_normalized(dt): return False diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index fcfbaa4ac2a1c..d7db37fc76460 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -679,7 +679,7 @@ def _add_offset(self, offset): values = self.tz_localize(None) else: values = self - result = offset.apply_index(values) + result = offset._apply_array(values) result = DatetimeArray._simple_new(result) result = result.tz_localize(self.tz) diff --git a/pandas/tests/tseries/offsets/test_offsets.py b/pandas/tests/tseries/offsets/test_offsets.py index 784c04f225630..b2bd69d7511e4 100644 --- a/pandas/tests/tseries/offsets/test_offsets.py +++ b/pandas/tests/tseries/offsets/test_offsets.py @@ -3663,14 +3663,19 @@ def test_offset(self, case): @pytest.mark.parametrize("case", offset_cases) def test_apply_index(self, case): + # https://github.com/pandas-dev/pandas/issues/34580 offset, cases = case s = DatetimeIndex(cases.keys()) + exp = DatetimeIndex(cases.values()) + with tm.assert_produces_warning(None): # GH#22535 check that we don't get a FutureWarning from adding # an integer array to PeriodIndex result = offset + s + tm.assert_index_equal(result, exp) - exp = DatetimeIndex(cases.values()) + with tm.assert_produces_warning(FutureWarning): + result = offset.apply_index(s) tm.assert_index_equal(result, exp) on_offset_cases = [ From aba0a9c67113416dd7799b891ee15885d9a66afc Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 7 Jul 2020 14:55:39 -0500 Subject: [PATCH 2/7] fixup --- pandas/_libs/tslibs/offsets.pyx | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pandas/_libs/tslibs/offsets.pyx b/pandas/_libs/tslibs/offsets.pyx index 113c9ce01c3ed..be021cd87b790 100644 --- a/pandas/_libs/tslibs/offsets.pyx +++ b/pandas/_libs/tslibs/offsets.pyx @@ -106,12 +106,6 @@ def apply_index_wraps(func): # do @functools.wraps(func) manually since it doesn't work on cdef funcs wrapper.__name__ = func.__name__ wrapper.__doc__ = func.__doc__ - try: - wrapper.__module__ = func.__module__ - except AttributeError: - # AttributeError: 'method_descriptor' object has no - # attribute '__module__' - pass return wrapper From 7c9110034e6bf38be2ed360528ccfbea709e82d4 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 7 Jul 2020 14:56:53 -0500 Subject: [PATCH 3/7] fixup --- pandas/_libs/tslibs/offsets.pyx | 34 ++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/pandas/_libs/tslibs/offsets.pyx b/pandas/_libs/tslibs/offsets.pyx index be021cd87b790..01c6b3418e226 100644 --- a/pandas/_libs/tslibs/offsets.pyx +++ b/pandas/_libs/tslibs/offsets.pyx @@ -594,7 +594,7 @@ cdef class BaseOffset: ) @apply_array_wraps - def _apply_array(self, dtindex): + def _apply_array(self, dtarr): raise NotImplementedError( f"DateOffset subclass {type(self).__name__} " "does not have a vectorized implementation" @@ -1067,8 +1067,8 @@ cdef class RelativeDeltaOffset(BaseOffset): return self._apply_array(dtindex) @apply_array_wraps - def _apply_array(self, dtindex): - dt64other = np.asarray(dtindex) + def _apply_array(self, dtarr): + dt64other = np.asarray(dtarr) kwds = self.kwds relativedelta_fast = { "years", @@ -1400,8 +1400,8 @@ cdef class BusinessDay(BusinessMixin): return self._apply_array(dtindex) @apply_array_wraps - def _apply_array(self, dtindex): - i8other = dtindex.view("i8") + def _apply_array(self, dtarr): + i8other = dtarr.view("i8") return shift_bdays(i8other, self.n) def is_on_offset(self, dt: datetime) -> bool: @@ -1887,9 +1887,9 @@ cdef class YearOffset(SingleConstructorOffset): return self._apply_array(dtindex) @apply_array_wraps - def _apply_array(self, dtindex): + def _apply_array(self, dtarr): shifted = shift_quarters( - dtindex.view("i8"), self.n, self.month, self._day_opt, modby=12 + dtarr.view("i8"), self.n, self.month, self._day_opt, modby=12 ) return shifted @@ -2044,9 +2044,9 @@ cdef class QuarterOffset(SingleConstructorOffset): return self._apply_array(dtindex) @apply_array_wraps - def _apply_array(self, dtindex): + def _apply_array(self, dtarr): shifted = shift_quarters( - dtindex.view("i8"), self.n, self.startingMonth, self._day_opt + dtarr.view("i8"), self.n, self.startingMonth, self._day_opt ) return shifted @@ -2163,8 +2163,8 @@ cdef class MonthOffset(SingleConstructorOffset): return self._apply_array(dtindex) @apply_array_wraps - def _apply_array(self, dtindex): - shifted = shift_months(dtindex.view("i8"), self.n, self._day_opt) + def _apply_array(self, dtarr): + shifted = shift_months(dtarr.view("i8"), self.n, self._day_opt) return shifted cpdef __setstate__(self, state): @@ -2306,9 +2306,9 @@ cdef class SemiMonthOffset(SingleConstructorOffset): @apply_array_wraps @cython.wraparound(False) @cython.boundscheck(False) - def _apply_array(self, dtindex): + def _apply_array(self, dtarr): cdef: - int64_t[:] i8other = dtindex.view("i8") + int64_t[:] i8other = dtarr.view("i8") Py_ssize_t i, count = len(i8other) int64_t val int64_t[:] out = np.empty(count, dtype="i8") @@ -2469,13 +2469,13 @@ cdef class Week(SingleConstructorOffset): return self._apply_array(dtindex) @apply_array_wraps - def _apply_array(self, dtindex): + def _apply_array(self, dtarr): if self.weekday is None: td = timedelta(days=7 * self.n) td64 = np.timedelta64(td, "ns") - return dtindex + td64 + return dtarr + td64 else: - i8other = dtindex.view("i8") + i8other = dtarr.view("i8") return self._end_apply_index(i8other) @cython.wraparound(False) @@ -3248,7 +3248,7 @@ cdef class CustomBusinessDay(BusinessDay): def apply_index(self, dtindex): raise NotImplementedError - def _apply_array(self, dtindex): + def _apply_array(self, dtarr): raise NotImplementedError def is_on_offset(self, dt: datetime) -> bool: From b44feb179bb599cfcf37ca10248c2d0d595256d4 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 14 Jul 2020 08:00:28 -0500 Subject: [PATCH 4/7] clean --- pandas/_libs/tslibs/offsets.pyx | 3 --- 1 file changed, 3 deletions(-) diff --git a/pandas/_libs/tslibs/offsets.pyx b/pandas/_libs/tslibs/offsets.pyx index 27537f03ec341..73e1b465746c4 100644 --- a/pandas/_libs/tslibs/offsets.pyx +++ b/pandas/_libs/tslibs/offsets.pyx @@ -109,9 +109,6 @@ def apply_index_wraps(func): "Use 'offset + other' instead.", FutureWarning) return result - # do @functools.wraps(func) manually since it doesn't work on cdef funcs - wrapper.__name__ = func.__name__ - wrapper.__doc__ = func.__doc__ return wrapper From d27af584544dda3a1f9764eb7ce7c3bdc5f12a23 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 14 Jul 2020 08:46:14 -0500 Subject: [PATCH 5/7] doc ref --- doc/source/reference/offset_frequency.rst | 17 +++++++++++++++++ pandas/_libs/tslibs/offsets.pyx | 6 ++++++ 2 files changed, 23 insertions(+) diff --git a/doc/source/reference/offset_frequency.rst b/doc/source/reference/offset_frequency.rst index 1b63253cde2c5..9e237b5823c04 100644 --- a/doc/source/reference/offset_frequency.rst +++ b/doc/source/reference/offset_frequency.rst @@ -33,6 +33,7 @@ Methods :toctree: api/ DateOffset.apply + DateOffset.apply_index DateOffset.copy DateOffset.isAnchored DateOffset.onOffset @@ -117,6 +118,7 @@ Methods :toctree: api/ BusinessHour.apply + BusinessHour.apply_index BusinessHour.copy BusinessHour.isAnchored BusinessHour.onOffset @@ -201,6 +203,7 @@ Methods :toctree: api/ CustomBusinessHour.apply + BusinessHour.apply_index CustomBusinessHour.copy CustomBusinessHour.isAnchored CustomBusinessHour.onOffset @@ -401,6 +404,7 @@ Methods :toctree: api/ CustomBusinessMonthEnd.apply + CustomBusinessMonthEnd.apply_index CustomBusinessMonthEnd.copy CustomBusinessMonthEnd.isAnchored CustomBusinessMonthEnd.onOffset @@ -447,6 +451,7 @@ Methods :toctree: api/ CustomBusinessMonthBegin.apply + CustomBusinessMonthBegin.apply_index CustomBusinessMonthBegin.copy CustomBusinessMonthBegin.isAnchored CustomBusinessMonthBegin.onOffset @@ -586,6 +591,7 @@ Methods :toctree: api/ WeekOfMonth.apply + WeekOfMonth.apply_index WeekOfMonth.copy WeekOfMonth.isAnchored WeekOfMonth.onOffset @@ -622,6 +628,7 @@ Methods :toctree: api/ LastWeekOfMonth.apply + LastWeekOfMonth.apply_index LastWeekOfMonth.copy LastWeekOfMonth.isAnchored LastWeekOfMonth.onOffset @@ -938,6 +945,7 @@ Methods :toctree: api/ FY5253.apply + FY5253.apply_index FY5253.copy FY5253.get_rule_code_suffix FY5253.get_year_end @@ -977,6 +985,7 @@ Methods :toctree: api/ FY5253Quarter.apply + FY5253Quarter.apply_index FY5253Quarter.copy FY5253Quarter.get_rule_code_suffix FY5253Quarter.get_weeks @@ -1013,6 +1022,7 @@ Methods :toctree: api/ Easter.apply + Easter.apply_index Easter.copy Easter.isAnchored Easter.onOffset @@ -1053,6 +1063,7 @@ Methods Tick.is_on_offset Tick.__call__ Tick.apply + Tick.apply_index Day --- @@ -1087,6 +1098,7 @@ Methods Day.is_on_offset Day.__call__ Day.apply + Day.apply_index Hour ---- @@ -1121,6 +1133,7 @@ Methods Hour.is_on_offset Hour.__call__ Hour.apply + Hour.apply_index Minute ------ @@ -1189,6 +1202,7 @@ Methods Second.is_on_offset Second.__call__ Second.apply + Second.apply_index Milli ----- @@ -1223,6 +1237,7 @@ Methods Milli.is_on_offset Milli.__call__ Milli.apply + Milli.apply_index Micro ----- @@ -1257,6 +1272,7 @@ Methods Micro.is_on_offset Micro.__call__ Micro.apply + Micro.apply_inex Nano ---- @@ -1291,6 +1307,7 @@ Methods Nano.is_on_offset Nano.__call__ Nano.apply + Nano.apply_index .. _api.frequencies: diff --git a/pandas/_libs/tslibs/offsets.pyx b/pandas/_libs/tslibs/offsets.pyx index 73e1b465746c4..0ef9381031420 100644 --- a/pandas/_libs/tslibs/offsets.pyx +++ b/pandas/_libs/tslibs/offsets.pyx @@ -545,6 +545,12 @@ cdef class BaseOffset: Returns ------- DatetimeIndex + + Raises + ------ + NotImplementedError + When the specific offset subclass does not have a vectorized + implementation. """ raise NotImplementedError( f"DateOffset subclass {type(self).__name__} " From 46b5b847016c7d9f8e4ad273d7e5144269b3689d Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 14 Jul 2020 09:08:52 -0500 Subject: [PATCH 6/7] fixup --- doc/source/reference/offset_frequency.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/reference/offset_frequency.rst b/doc/source/reference/offset_frequency.rst index 9e237b5823c04..04d5f94ed9540 100644 --- a/doc/source/reference/offset_frequency.rst +++ b/doc/source/reference/offset_frequency.rst @@ -1272,7 +1272,7 @@ Methods Micro.is_on_offset Micro.__call__ Micro.apply - Micro.apply_inex + Micro.apply_index Nano ---- From 0189b9befc5216968f1453fd1d85a0b386e363f4 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 14 Jul 2020 14:44:20 -0500 Subject: [PATCH 7/7] fixups --- doc/source/reference/offset_frequency.rst | 3 ++- pandas/_libs/tslibs/offsets.pyx | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/source/reference/offset_frequency.rst b/doc/source/reference/offset_frequency.rst index 04d5f94ed9540..e6271a7806706 100644 --- a/doc/source/reference/offset_frequency.rst +++ b/doc/source/reference/offset_frequency.rst @@ -203,7 +203,7 @@ Methods :toctree: api/ CustomBusinessHour.apply - BusinessHour.apply_index + CustomBusinessHour.apply_index CustomBusinessHour.copy CustomBusinessHour.isAnchored CustomBusinessHour.onOffset @@ -1168,6 +1168,7 @@ Methods Minute.is_on_offset Minute.__call__ Minute.apply + Minute.apply_index Second ------ diff --git a/pandas/_libs/tslibs/offsets.pyx b/pandas/_libs/tslibs/offsets.pyx index 0ef9381031420..9a7ca15a2a1c2 100644 --- a/pandas/_libs/tslibs/offsets.pyx +++ b/pandas/_libs/tslibs/offsets.pyx @@ -86,7 +86,7 @@ cdef bint _is_normalized(datetime dt): return True -def apply_wrapper_core(func, self, other): +def apply_wrapper_core(func, self, other) -> ndarray: result = func(self, other) result = np.asarray(result)