BUG: Bug in DataFrame construction with nulls and datetimes in a list like

closes #15869

Author: Jeff Reback <[email protected]>

Closes #15892 from jreback/construct and squashes the following commits:

6bf2148 [Jeff Reback] fix perf
7fcd4e5 [Jeff Reback] BUG: Bug in DataFrame construction with nulls and datetimes in a list-like

Author: jreback

doc/source/whatsnew/v0.20.0.txt

@@ -997,6 +997,7 @@ Conversion
 - Bug in ``Series.ffill()`` with mixed dtypes containing tz-aware datetimes. (:issue:`14956`)
 - Bug in ``DataFrame.fillna()`` where the argument ``downcast`` was ignored when fillna value was of type ``dict`` (:issue:`15277`)
 - Bug in ``.asfreq()``, where frequency was not set for empty ``Series`` (:issue:`14320`)
+- Bug in ``DataFrame`` construction with nulls and datetimes in a list-like (:issue:`15869`)
 
 Indexing
 ^^^^^^^^

pandas/_libs/src/inference.pyx

@@ -439,31 +439,86 @@ def infer_dtype(object value):
     return 'mixed'
 
 
-cpdef bint is_possible_datetimelike_array(object arr):
-    # determine if we have a possible datetimelike (or null-like) array
+cpdef object infer_datetimelike_array(object arr):
+    """
+    infer if we have a datetime or timedelta array
+    - date: we have *only* date and maybe strings, nulls
+    - datetime: we have *only* datetimes and maybe strings, nulls
+    - timedelta: we have *only* timedeltas and maybe strings, nulls
+    - nat: we do not have *any* date, datetimes or timedeltas, but do have
+      at least a NaT
+    - mixed: other objects (strings or actual objects)
+
+    Parameters
+    ----------
+    arr : object array
+
+    Returns
+    -------
+    string: {datetime, timedelta, date, nat, mixed}
+
+    """
+
     cdef:
         Py_ssize_t i, n = len(arr)
-        bint seen_timedelta = 0, seen_datetime = 0
+        bint seen_timedelta = 0, seen_date = 0, seen_datetime = 0
+        bint seen_nat = 0
+        list objs = []
         object v
 
     for i in range(n):
         v = arr[i]
         if util.is_string_object(v):
-            continue
+            objs.append(v)
+
+            if len(objs) == 3:
+                break
+
         elif util._checknull(v):
-            continue
-        elif is_datetime(v):
-            seen_datetime=1
-        elif is_timedelta(v):
-            seen_timedelta=1
+            # nan or None
+            pass
+        elif v is NaT:
+            seen_nat = 1
+        elif is_datetime(v) or util.is_datetime64_object(v):
+            # datetime, or np.datetime64
+            seen_datetime = 1
+        elif is_date(v):
+            seen_date = 1
+        elif is_timedelta(v) or util.is_timedelta64_object(v):
+            # timedelta, or timedelta64
+            seen_timedelta = 1
         else:
-            return False
-    return seen_datetime or seen_timedelta
+            return 'mixed'
+
+    if seen_date and not (seen_datetime or seen_timedelta):
+        return 'date'
+    elif seen_datetime and not seen_timedelta:
+        return 'datetime'
+    elif seen_timedelta and not seen_datetime:
+        return 'timedelta'
+    elif seen_nat:
+        return 'nat'
+
+    # short-circuit by trying to
+    # actually convert these strings
+    # this is for performance as we don't need to try
+    # convert *every* string array
+    if len(objs):
+        try:
+            tslib.array_to_datetime(objs, errors='raise')
+            return 'datetime'
+        except:
+            pass
+
+        # we are *not* going to infer from strings
+        # for timedelta as too much ambiguity
+
+    return 'mixed'
 
 
 cdef inline bint is_null_datetimelike(v):
     # determine if we have a null for a timedelta/datetime (or integer
-    # versions)x
+    # versions)
     if util._checknull(v):
         return True
     elif v is NaT:

pandas/tests/frame/test_constructors.py

@@ -1366,6 +1366,15 @@ def test_constructor_with_datetimes(self):
                               .reset_index(drop=True), 'b': i_no_tz})
         tm.assert_frame_equal(df, expected)
 
+    def test_constructor_datetimes_with_nulls(self):
+        # gh-15869
+        for arr in [np.array([None, None, None, None,
+                              datetime.now(), None]),
+                    np.array([None, None, datetime.now(), None])]:
+            result = DataFrame(arr).get_dtype_counts()
+            expected = Series({'datetime64[ns]': 1})
+            tm.assert_series_equal(result, expected)
+
     def test_constructor_for_list_with_dtypes(self):
         # TODO(wesm): unused
         intname = np.dtype(np.int_).name  # noqa

pandas/tests/frame/test_misc_api.py

@@ -12,7 +12,7 @@
 from numpy.random import randn
 import numpy as np
 
-from pandas import DataFrame, Series
+from pandas import DataFrame, Series, date_range, timedelta_range
 import pandas as pd
 
 from pandas.util.testing import (assert_almost_equal,
@@ -328,6 +328,16 @@ def test_empty_nonzero(self):
             self.assertTrue(df.empty)
             self.assertTrue(df.T.empty)
 
+    def test_with_datetimelikes(self):
+
+        df = DataFrame({'A': date_range('20130101', periods=10),
+                        'B': timedelta_range('1 day', periods=10)})
+        t = df.T
+
+        result = t.get_dtype_counts()
+        expected = Series({'object': 10})
+        tm.assert_series_equal(result, expected)
+
     def test_inplace_return_self(self):
         # re #1893
 

pandas/tests/series/test_constructors.py

@@ -327,6 +327,14 @@ def test_constructor_datelike_coercion(self):
         result = df.loc['216']
         self.assertTrue(result.dtype == object)
 
+    def test_constructor_datetimes_with_nulls(self):
+        # gh-15869
+        for arr in [np.array([None, None, None, None,
+                              datetime.now(), None]),
+                    np.array([None, None, datetime.now(), None])]:
+            result = Series(arr)
+            assert result.dtype == 'M8[ns]'
+
     def test_constructor_dtype_datetime64(self):
 
         s = Series(iNaT, dtype='M8[ns]', index=lrange(5))

pandas/types/cast.py

@@ -748,8 +748,6 @@ def maybe_infer_to_datetimelike(value, convert_dates=False):
     this is pretty strict in that a datetime/timedelta is REQUIRED
     in addition to possible nulls/string likes
 
-    ONLY strings are NOT datetimelike
-
     Parameters
     ----------
     value : np.array / Series / Index / list-like
@@ -770,64 +768,70 @@ def maybe_infer_to_datetimelike(value, convert_dates=False):
     if not is_list_like(v):
         v = [v]
     v = np.array(v, copy=False)
+
+    # we only care about object dtypes
+    if not is_object_dtype(v):
+        return value
+
     shape = v.shape
     if not v.ndim == 1:
         v = v.ravel()
 
-    if len(v):
-
-        def _try_datetime(v):
-            # safe coerce to datetime64
-            try:
-                v = tslib.array_to_datetime(v, errors='raise')
-            except ValueError:
+    if not len(v):
+        return value
 
-                # we might have a sequence of the same-datetimes with tz's
-                # if so coerce to a DatetimeIndex; if they are not the same,
-                # then these stay as object dtype
-                try:
-                    from pandas import to_datetime
-                    return to_datetime(v)
-                except:
-                    pass
+    def try_datetime(v):
+        # safe coerce to datetime64
+        try:
+            v = tslib.array_to_datetime(v, errors='raise')
+        except ValueError:
 
+            # we might have a sequence of the same-datetimes with tz's
+            # if so coerce to a DatetimeIndex; if they are not the same,
+            # then these stay as object dtype
+            try:
+                from pandas import to_datetime
+                return to_datetime(v)
             except:
                 pass
 
-            return v.reshape(shape)
+        except:
+            pass
 
-        def _try_timedelta(v):
-            # safe coerce to timedelta64
+        return v.reshape(shape)
 
-            # will try first with a string & object conversion
-            from pandas import to_timedelta
-            try:
-                return to_timedelta(v)._values.reshape(shape)
-            except:
-                return v
-
-        # do a quick inference for perf
-        sample = v[:min(3, len(v))]
-        inferred_type = lib.infer_dtype(sample)
-
-        if (inferred_type in ['datetime', 'datetime64'] or
-                (convert_dates and inferred_type in ['date'])):
-            value = _try_datetime(v)
-        elif inferred_type in ['timedelta', 'timedelta64']:
-            value = _try_timedelta(v)
-
-        # It's possible to have nulls intermixed within the datetime or
-        # timedelta.  These will in general have an inferred_type of 'mixed',
-        # so have to try both datetime and timedelta.
-
-        # try timedelta first to avoid spurious datetime conversions
-        # e.g. '00:00:01' is a timedelta but technically is also a datetime
-        elif inferred_type in ['mixed']:
-
-            if lib.is_possible_datetimelike_array(_ensure_object(v)):
-                value = _try_timedelta(v)
-                if lib.infer_dtype(value) in ['mixed']:
-                    value = _try_datetime(v)
+    def try_timedelta(v):
+        # safe coerce to timedelta64
+
+        # will try first with a string & object conversion
+        from pandas import to_timedelta
+        try:
+            return to_timedelta(v)._values.reshape(shape)
+        except:
+            return v
+
+    inferred_type = lib.infer_datetimelike_array(_ensure_object(v))
+
+    if inferred_type == 'date' and convert_dates:
+        value = try_datetime(v)
+    elif inferred_type == 'datetime':
+        value = try_datetime(v)
+    elif inferred_type == 'timedelta':
+        value = try_timedelta(v)
+    elif inferred_type == 'nat':
+
+        # if all NaT, return as datetime
+        if isnull(v).all():
+            value = try_datetime(v)
+        else:
+
+            # We have at least a NaT and a string
+            # try timedelta first to avoid spurious datetime conversions
+            # e.g. '00:00:01' is a timedelta but
+            # technically is also a datetime
+            value = try_timedelta(v)
+            if lib.infer_dtype(value) in ['mixed']:
+                value = try_datetime(v)
 
     return value