updated whatsnew and simplified to_timestamp(how='end')

Author: reidy-p

doc/source/whatsnew/v0.23.0.txt

@@ -193,6 +193,60 @@ If installed, we now require:
    | openpyxl        | 2.4.0           |          |
    +-----------------+-----------------+----------+
 
+.. _whatsnew_0230.api_breaking.end_time:
+
+Time values in ``dt.end_time`` and ``to_timestamp(how='end')``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The time values in ``Period`` and ``PeriodIndex`` objects are now adjusted
+appropriately when calling :attr:`Series.dt.end_time`, :attr:`Period.end_time`,
+:attr:`PeriodIndex.end_time`, :func:`Period.to_timestamp()` with ``how='end'``,
+and :func:`PeriodIndex.to_timestamp()` with ``how='end'`` (:issue:`17157`)
+
+Previous Behavior:
+
+.. code-block:: ipython
+
+   In [2]: p = pd.Period('2017-01-01', 'D')
+   In [3]: pi = pd.PeriodIndex([p])
+
+   In [4]: pd.Series(p).dt.end_time[0]
+   Out[4]: Timestamp(2017-01-01 00:00:00)
+
+   In [5]: pd.Series(pi).dt.end_time[0]
+   Out[5]: Timestamp(2017-01-01 00:00:00)
+
+   In [6]: p.end_time
+   Out[6]: Timestamp(2017-01-01 23:59:59.999999999)
+
+   In [7]: pi.end_time[0]
+   Out[7]: Timestamp(2017-01-01 00:00:00)
+
+   In [8]: p.to_timestamp(how='end')
+   Out[8]: Timestamp(2017-01-01 00:00:00)
+
+   In [9]: pi.to_timestamp(how='end')[0]
+   Out[9]: Timestamp(2017-01-01 00:00:00)
+
+Current Behavior
+
+.. ipython:: python
+
+   p = pd.Period('2017-01-01', 'D')
+   pi = pd.PeriodIndex([p])
+
+   pd.Series(p).dt.end_time[0]
+
+   pd.Series(pi).dt.end_time[0]
+
+   p.end_time
+
+   pi.end_time[0]
+
+   p.to_timestamp(how='end')
+
+   pi.to_timestamp(how='end')[0]
+
 
 Build Changes
 ^^^^^^^^^^^^^
@@ -318,7 +372,6 @@ Documentation Changes
 Bug Fixes
 ~~~~~~~~~
 
-
 Conversion
 ^^^^^^^^^^
 
@@ -338,7 +391,6 @@ Conversion
 - Bug in :class:`Series` floor-division where operating on a scalar ``timedelta`` raises an exception (:issue:`18846`)
 - Bug in :class:`FY5253Quarter`, :class:`LastWeekOfMonth` where rollback and rollforward behavior was inconsistent with addition and subtraction behavior (:issue:`18854`)
 - Bug in :class:`Index` constructor with ``dtype=CategoricalDtype(...)`` where ``categories`` and ``ordered`` are not maintained (issue:`19032`)
-- Bug in :class:`Index` constructor with ``dtype=CategoricalDtype(...)`` where ``categories`` and ``ordered`` are not maintained (issue:`19032`) 
 - Bug in :class:`Series`` with ``dtype='timedelta64[ns]`` where addition or subtraction of ``TimedeltaIndex`` had results cast to ``dtype='int64'`` (:issue:`17250`)
 - Bug in :class:`TimedeltaIndex` where division by a ``Series`` would return a ``TimedeltaIndex`` instead of a ``Series`` (issue:`19042`)
 - Bug in :class:`Series` with ``dtype='timedelta64[ns]`` where addition or subtraction of ``TimedeltaIndex`` could return a ``Series`` with an incorrect name (issue:`19043`)

pandas/_libs/tslibs/period.pyx

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # cython: profile=False
+import pandas as pd
 from datetime import datetime, date, timedelta
 
 from cpython cimport (
@@ -756,6 +757,10 @@ cdef class _Period(object):
             freq = self._maybe_convert_freq(freq)
         how = _validate_end_alias(how)
 
+        end = how == 'E'
+        if end:
+            return (self + 1).to_timestamp(how='start') - pd.Timedelta(1, 'ns')
+
         if freq is None:
             base, mult = get_freq_code(self.freq)
             freq = get_to_timestamp_base(base)

pandas/core/indexes/period.py

@@ -2,6 +2,7 @@
 from datetime import datetime, timedelta
 import numpy as np
 import warnings
+import pandas as pd
 
 from pandas.core import common as com
 from pandas.core.dtypes.common import (
@@ -656,6 +657,10 @@ def to_timestamp(self, freq=None, how='start'):
         """
         how = _validate_end_alias(how)
 
+        end = how == 'E'
+        if end:
+            return (self + 1).to_timestamp(how='start') - pd.Timedelta(1, 'ns')
+
         if freq is None:
             base, mult = _gfc(self.freq)
             freq = frequencies.get_to_timestamp_base(base)
@@ -665,16 +670,7 @@ def to_timestamp(self, freq=None, how='start'):
         base, mult = _gfc(freq)
         new_data = self.asfreq(freq, how)
 
-        end = how == 'E'
-        if end:
-            indexer = np.where(new_data.notnull())
-            # move forward one period
-            new_data._values[indexer] += 1
-            new_data = period.periodarr_to_dt64arr(new_data._values, base)
-            # subtract one nanosecond
-            new_data[indexer] -= 1
-        else:
-            new_data = period.periodarr_to_dt64arr(new_data._values, base)
+        new_data = period.periodarr_to_dt64arr(new_data._values, base)
         return DatetimeIndex(new_data, freq='infer', name=self.name)
 
     def _maybe_convert_timedelta(self, other):

pandas/tests/indexes/period/test_tools.py

@@ -173,6 +173,7 @@ def test_to_timestamp(self):
 
         exp_index = date_range('1/1/2001', end='12/31/2009', freq='A-DEC')
         result = series.to_timestamp(how='end')
+        exp_index = exp_index + pd.Timedelta(1, 'D') - pd.Timedelta(1, 'ns')
         tm.assert_index_equal(result.index, exp_index)
         assert result.name == 'foo'
 
@@ -187,16 +188,19 @@ def _get_with_delta(delta, freq='A-DEC'):
         delta = timedelta(hours=23)
         result = series.to_timestamp('H', 'end')
         exp_index = _get_with_delta(delta)
+        exp_index = exp_index + pd.Timedelta(1, 'h') - pd.Timedelta(1, 'ns')
         tm.assert_index_equal(result.index, exp_index)
 
         delta = timedelta(hours=23, minutes=59)
         result = series.to_timestamp('T', 'end')
         exp_index = _get_with_delta(delta)
+        exp_index = exp_index + pd.Timedelta(1, 'm') - pd.Timedelta(1, 'ns')
         tm.assert_index_equal(result.index, exp_index)
 
         result = series.to_timestamp('S', 'end')
         delta = timedelta(hours=23, minutes=59, seconds=59)
         exp_index = _get_with_delta(delta)
+        exp_index = exp_index + pd.Timedelta(1, 's') - pd.Timedelta(1, 'ns')
         tm.assert_index_equal(result.index, exp_index)
 
         index = PeriodIndex(freq='H', start='1/1/2001', end='1/2/2001')
@@ -205,6 +209,7 @@ def _get_with_delta(delta, freq='A-DEC'):
         exp_index = date_range('1/1/2001 00:59:59', end='1/2/2001 00:59:59',
                                freq='H')
         result = series.to_timestamp(how='end')
+        exp_index = exp_index + pd.Timedelta(1, 's') - pd.Timedelta(1, 'ns')
         tm.assert_index_equal(result.index, exp_index)
         assert result.name == 'foo'
 
@@ -268,6 +273,7 @@ def test_to_timestamp_pi_mult(self):
         result = idx.to_timestamp(how='E')
         expected = DatetimeIndex(
             ['2011-02-28', 'NaT', '2011-03-31'], name='idx')
+        expected = expected + pd.Timedelta(1, 'D') - pd.Timedelta(1, 'ns')
         tm.assert_index_equal(result, expected)
 
     def test_to_timestamp_pi_combined(self):
@@ -278,11 +284,13 @@ def test_to_timestamp_pi_combined(self):
         tm.assert_index_equal(result, expected)
         result = idx.to_timestamp(how='E')
         expected = DatetimeIndex(
-            ['2011-01-02 00:59:59', '2011-01-03 01:59:59'], name='idx')
+            ['2011-01-02 01:00:00', '2011-01-03 02:00:00'], name='idx')
+        expected = expected - pd.Timedelta(1, 'ns')
         tm.assert_index_equal(result, expected)
         result = idx.to_timestamp(how='E', freq='H')
         expected = DatetimeIndex(
-            ['2011-01-02 00:00', '2011-01-03 01:00'], name='idx')
+            ['2011-01-02 01:00', '2011-01-03 02:00'], name='idx')
+        expected = expected - pd.Timedelta(1, 'ns')
         tm.assert_index_equal(result, expected)
 
     def test_to_timestamp_to_period_astype(self):
@@ -324,6 +332,7 @@ def test_period_astype_to_timestamp(self):
         tm.assert_index_equal(pi.astype('datetime64[ns]'), exp)
 
         exp = pd.DatetimeIndex(['2011-01-31', '2011-02-28', '2011-03-31'])
+        exp = exp + pd.Timedelta(1, 'D') - pd.Timedelta(1, 'ns')
         tm.assert_index_equal(pi.astype('datetime64[ns]', how='end'), exp)
 
         exp = pd.DatetimeIndex(['2011-01-01', '2011-02-01', '2011-03-01'],
@@ -333,6 +342,7 @@ def test_period_astype_to_timestamp(self):
 
         exp = pd.DatetimeIndex(['2011-01-31', '2011-02-28', '2011-03-31'],
                                tz='US/Eastern')
+        exp = exp + pd.Timedelta(1, 'D') - pd.Timedelta(1, 'ns')
         res = pi.astype('datetime64[ns, US/Eastern]', how='end')
         tm.assert_index_equal(res, exp)