BUG/API: to_datetime preserves UTC offsets when parsing datetime strings (#21822)

Author: mroeschke

asv_bench/benchmarks/timeseries.py

@@ -343,6 +343,25 @@ def time_iso8601_tz_spaceformat(self):
         to_datetime(self.strings_tz_space)
 
 
+class ToDatetimeNONISO8601(object):
+
+    goal_time = 0.2
+
+    def setup(self):
+        N = 10000
+        half = int(N / 2)
+        ts_string_1 = 'March 1, 2018 12:00:00+0400'
+        ts_string_2 = 'March 1, 2018 12:00:00+0500'
+        self.same_offset = [ts_string_1] * N
+        self.diff_offset = [ts_string_1] * half + [ts_string_2] * half
+
+    def time_same_offset(self):
+        to_datetime(self.same_offset)
+
+    def time_different_offset(self):
+        to_datetime(self.diff_offset)
+
+
 class ToDatetimeFormat(object):
 
     goal_time = 0.2

doc/source/whatsnew/v0.24.0.txt

@@ -224,6 +224,62 @@ For situations where you need an ``ndarray`` of ``Interval`` objects, use
    np.asarray(idx)
    idx.values.astype(object)
 
+.. _whatsnew_0240.api.timezone_offset_parsing:
+
+Parsing Datetime Strings with Timezone Offsets
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Previously, parsing datetime strings with UTC offsets with :func:`to_datetime`
+or :class:`DatetimeIndex` would automatically convert the datetime to UTC
+without timezone localization. This is inconsistent from parsing the same
+datetime string with :class:`Timestamp` which would preserve the UTC
+offset in the ``tz`` attribute. Now, :func:`to_datetime` preserves the UTC
+offset in the ``tz`` attribute when all the datetime strings have the same
+UTC offset (:issue:`17697`, :issue:`11736`)
+
+*Previous Behavior*:
+
+.. code-block:: ipython
+
+
+    In [2]: pd.to_datetime("2015-11-18 15:30:00+05:30")
+    Out[2]: Timestamp('2015-11-18 10:00:00')
+
+    In [3]: pd.Timestamp("2015-11-18 15:30:00+05:30")
+    Out[3]: Timestamp('2015-11-18 15:30:00+0530', tz='pytz.FixedOffset(330)')
+
+    # Different UTC offsets would automatically convert the datetimes to UTC (without a UTC timezone)
+    In [4]: pd.to_datetime(["2015-11-18 15:30:00+05:30", "2015-11-18 16:30:00+06:30"])
+    Out[4]: DatetimeIndex(['2015-11-18 10:00:00', '2015-11-18 10:00:00'], dtype='datetime64[ns]', freq=None)
+
+*Current Behavior*:
+
+.. ipython:: python
+
+    pd.to_datetime("2015-11-18 15:30:00+05:30")
+    pd.Timestamp("2015-11-18 15:30:00+05:30")
+
+Parsing datetime strings with the same UTC offset will preserve the UTC offset in the ``tz``
+
+.. ipython:: python
+
+    pd.to_datetime(["2015-11-18 15:30:00+05:30"] * 2)
+
+Parsing datetime strings with different UTC offsets will now create an Index of
+``datetime.datetime`` objects with different UTC offsets
+
+.. ipython:: python
+
+    idx = pd.to_datetime(["2015-11-18 15:30:00+05:30", "2015-11-18 16:30:00+06:30"])
+    idx
+    idx[0]
+    idx[1]
+
+Passing ``utc=True`` will mimic the previous behavior but will correctly indicate
+that the dates have been converted to UTC
+
+.. ipython:: python
+    pd.to_datetime(["2015-11-18 15:30:00+05:30", "2015-11-18 16:30:00+06:30"], utc=True)
 
 .. _whatsnew_0240.api.datetimelike.normalize:
 
@@ -439,6 +495,7 @@ Datetimelike
 
 - Fixed bug where two :class:`DateOffset` objects with different ``normalize`` attributes could evaluate as equal (:issue:`21404`)
 - Fixed bug where :meth:`Timestamp.resolution` incorrectly returned 1-microsecond ``timedelta`` instead of 1-nanosecond :class:`Timedelta` (:issue:`21336`,:issue:`21365`)
+- Bug in :func:`to_datetime` that did not consistently return an :class:`Index` when ``box=True`` was specified (:issue:`21864`)
 
 Timedelta
 ^^^^^^^^^

pandas/_libs/tslib.pyx

@@ -19,6 +19,7 @@ import numpy as np
 cnp.import_array()
 
 import pytz
+from dateutil.tz import tzlocal, tzutc as dateutil_utc
 
 
 from util cimport (is_integer_object, is_float_object, is_string_object,
@@ -328,7 +329,7 @@ cpdef array_with_unit_to_datetime(ndarray values, unit, errors='coerce'):
     if unit == 'ns':
         if issubclass(values.dtype.type, np.integer):
             return values.astype('M8[ns]')
-        return array_to_datetime(values.astype(object), errors=errors)
+        return array_to_datetime(values.astype(object), errors=errors)[0]
 
     m = cast_from_unit(None, unit)
 
@@ -457,21 +458,58 @@ cpdef array_to_datetime(ndarray[object] values, errors='raise',
                         dayfirst=False, yearfirst=False,
                         format=None, utc=None,
                         require_iso8601=False):
+    """
+    Converts a 1D array of date-like values to a numpy array of either:
+        1) datetime64[ns] data
+        2) datetime.datetime objects, if OutOfBoundsDatetime or TypeError
+           is encountered
+
+    Also returns a pytz.FixedOffset if an array of strings with the same
+    timezone offset is passed and utc=True is not passed. Otherwise, None
+    is returned
+
+    Handles datetime.date, datetime.datetime, np.datetime64 objects, numeric,
+    strings
+
+    Parameters
+    ----------
+    values : ndarray of object
+         date-like objects to convert
+    errors : str, default 'raise'
+         error behavior when parsing
+    dayfirst : bool, default False
+         dayfirst parsing behavior when encountering datetime strings
+    yearfirst : bool, default False
+         yearfirst parsing behavior when encountering datetime strings
+    format : str, default None
+         format of the string to parse
+    utc : bool, default None
+         indicator whether the dates should be UTC
+    require_iso8601 : bool, default False
+         indicator whether the datetime string should be iso8601
+
+    Returns
+    -------
+    tuple (ndarray, tzoffset)
+    """
     cdef:
         Py_ssize_t i, n = len(values)
-        object val, py_dt
+        object val, py_dt, tz, tz_out = None
         ndarray[int64_t] iresult
         ndarray[object] oresult
         npy_datetimestruct dts
         bint utc_convert = bool(utc)
         bint seen_integer = 0
         bint seen_string = 0
         bint seen_datetime = 0
+        bint seen_datetime_offset = 0
         bint is_raise = errors=='raise'
         bint is_ignore = errors=='ignore'
         bint is_coerce = errors=='coerce'
         _TSObject _ts
         int out_local=0, out_tzoffset=0
+        float offset_seconds
+        set out_tzoffset_vals = set()
 
     # specify error conditions
     assert is_raise or is_ignore or is_coerce
@@ -584,7 +622,7 @@ cpdef array_to_datetime(ndarray[object] values, errors='raise',
                             raise ValueError("time data {val} doesn't match "
                                              "format specified"
                                              .format(val=val))
-                        return values
+                        return values, tz_out
 
                     try:
                         py_dt = parse_datetime_string(val, dayfirst=dayfirst,
@@ -595,6 +633,30 @@ cpdef array_to_datetime(ndarray[object] values, errors='raise',
                             continue
                         raise TypeError("invalid string coercion to datetime")
 
+                    # If the dateutil parser returned tzinfo, capture it
+                    # to check if all arguments have the same tzinfo
+                    tz = py_dt.tzinfo
+                    if tz is not None:
+                        seen_datetime_offset = 1
+                        if tz == dateutil_utc():
+                            # dateutil.tz.tzutc has no offset-like attribute
+                            # Just add the 0 offset explicitly
+                            out_tzoffset_vals.add(0)
+                        elif tz == tzlocal():
+                            # is comparison fails unlike other dateutil.tz
+                            # objects. Also, dateutil.tz.tzlocal has no
+                            # _offset attribute like tzoffset
+                            offset_seconds = tz._dst_offset.total_seconds()
+                            out_tzoffset_vals.add(offset_seconds)
+                        else:
+                            # dateutil.tz.tzoffset objects cannot be hashed
+                            # store the total_seconds() instead
+                            offset_seconds = tz._offset.total_seconds()
+                            out_tzoffset_vals.add(offset_seconds)
+                    else:
+                        # Add a marker for naive string, to track if we are
+                        # parsing mixed naive and aware strings
+                        out_tzoffset_vals.add('naive')
                     try:
                         _ts = convert_datetime_to_tsobject(py_dt, None)
                         iresult[i] = _ts.value
@@ -614,8 +676,17 @@ cpdef array_to_datetime(ndarray[object] values, errors='raise',
                     # where we left off
                     value = dtstruct_to_dt64(&dts)
                     if out_local == 1:
+                        seen_datetime_offset = 1
+                        # Store the out_tzoffset in seconds
+                        # since we store the total_seconds of
+                        # dateutil.tz.tzoffset objects
+                        out_tzoffset_vals.add(out_tzoffset * 60.)
                         tz = pytz.FixedOffset(out_tzoffset)
                         value = tz_convert_single(value, tz, 'UTC')
+                    else:
+                        # Add a marker for naive string, to track if we are
+                        # parsing mixed naive and aware strings
+                        out_tzoffset_vals.add('naive')
                     iresult[i] = value
                     try:
                         check_dts_bounds(&dts)
@@ -631,7 +702,7 @@ cpdef array_to_datetime(ndarray[object] values, errors='raise',
                                 raise ValueError("time data {val} doesn't "
                                                  "match format specified"
                                                  .format(val=val))
-                            return values
+                            return values, tz_out
                         raise
 
             else:
@@ -657,7 +728,21 @@ cpdef array_to_datetime(ndarray[object] values, errors='raise',
             else:
                 raise TypeError
 
-        return result
+        if seen_datetime_offset and not utc_convert:
+            # GH 17697
+            # 1) If all the offsets are equal, return one offset for
+            #    the parsed dates to (maybe) pass to DatetimeIndex
+            # 2) If the offsets are different, then force the parsing down the
+            #    object path where an array of datetimes
+            #    (with individual dateutil.tzoffsets) are returned
+            is_same_offsets = len(out_tzoffset_vals) == 1
+            if not is_same_offsets:
+                return array_to_datetime_object(values, is_raise,
+                                                dayfirst, yearfirst)
+            else:
+                tz_offset = out_tzoffset_vals.pop()
+                tz_out = pytz.FixedOffset(tz_offset / 60.)
+        return result, tz_out
     except OutOfBoundsDatetime:
         if is_raise:
             raise
@@ -679,36 +764,67 @@ cpdef array_to_datetime(ndarray[object] values, errors='raise',
                     oresult[i] = val.item()
             else:
                 oresult[i] = val
-        return oresult
+        return oresult, tz_out
     except TypeError:
-        oresult = np.empty(n, dtype=object)
+        return array_to_datetime_object(values, is_raise, dayfirst, yearfirst)
 
-        for i in range(n):
-            val = values[i]
-            if checknull_with_nat(val):
-                oresult[i] = val
-            elif is_string_object(val):
 
-                if len(val) == 0 or val in nat_strings:
-                    oresult[i] = 'NaT'
-                    continue
+cdef array_to_datetime_object(ndarray[object] values, bint is_raise,
+                              dayfirst=False, yearfirst=False):
+    """
+    Fall back function for array_to_datetime
 
-                try:
-                    oresult[i] = parse_datetime_string(val, dayfirst=dayfirst,
-                                                       yearfirst=yearfirst)
-                    pydatetime_to_dt64(oresult[i], &dts)
-                    check_dts_bounds(&dts)
-                except Exception:
-                    if is_raise:
-                        raise
-                    return values
-                    # oresult[i] = val
-            else:
+    Attempts to parse datetime strings with dateutil to return an array
+    of datetime objects
+
+    Parameters
+    ----------
+    values : ndarray of object
+         date-like objects to convert
+    is_raise : bool
+         error behavior when parsing
+    dayfirst : bool, default False
+         dayfirst parsing behavior when encountering datetime strings
+    yearfirst : bool, default False
+         yearfirst parsing behavior when encountering datetime strings
+
+    Returns
+    -------
+    tuple (ndarray, None)
+    """
+    cdef:
+        Py_ssize_t i, n = len(values)
+        object val,
+        ndarray[object] oresult
+        npy_datetimestruct dts
+
+    oresult = np.empty(n, dtype=object)
+
+    # We return an object array and only attempt to parse:
+    # 1) NaT or NaT-like values
+    # 2) datetime strings, which we return as datetime.datetime
+    for i in range(n):
+        val = values[i]
+        if checknull_with_nat(val):
+            oresult[i] = val
+        elif is_string_object(val):
+            if len(val) == 0 or val in nat_strings:
+                oresult[i] = 'NaT'
+                continue
+            try:
+                oresult[i] = parse_datetime_string(val, dayfirst=dayfirst,
+                                                   yearfirst=yearfirst)
+                pydatetime_to_dt64(oresult[i], &dts)
+                check_dts_bounds(&dts)
+            except (ValueError, OverflowError):
                 if is_raise:
                     raise
-                return values
-
-        return oresult
+                return values, None
+        else:
+            if is_raise:
+                raise
+            return values, None
+    return oresult, None
 
 
 cdef inline bint _parse_today_now(str val, int64_t* iresult):

pandas/core/dtypes/cast.py

@@ -918,7 +918,7 @@ def try_datetime(v):
             # GH19671
             v = tslib.array_to_datetime(v,
                                         require_iso8601=True,
-                                        errors='raise')
+                                        errors='raise')[0]
         except ValueError:
 
             # we might have a sequence of the same-datetimes with tz's