BUG: upconvert datetime64 to datetime.datetime when necessary in concat steps. close #2624

Author: wesm

RELEASE.rst

@@ -100,6 +100,7 @@ pandas 0.10.1
   - Don't lose time zone when calling DatetimeIndex.drop (GH2621_)
   - Fix setitem on a Series with a boolean key and a non-scalar as value (GH2686_)
   - Box datetime64 values in Series.apply/map (GH2627_, GH2689_)
+  - Upconvert datetime + datetime64 values when concatenating frames (GH2624_)
 
 **API Changes**
 
@@ -120,6 +121,7 @@ pandas 0.10.1
 .. _GH2613: https://github.com/pydata/pandas/issues/2613
 .. _GH2616: https://github.com/pydata/pandas/issues/2616
 .. _GH2621: https://github.com/pydata/pandas/issues/2621
+.. _GH2624: https://github.com/pydata/pandas/issues/2624
 .. _GH2625: https://github.com/pydata/pandas/issues/2625
 .. _GH2627: https://github.com/pydata/pandas/issues/2627
 .. _GH2643: https://github.com/pydata/pandas/issues/2643

pandas/core/common.py

@@ -1216,13 +1216,26 @@ def _concat_compat(to_concat, axis=0):
     # filter empty arrays
     to_concat = [x for x in to_concat if x.shape[axis] > 0]
 
-    if all(x.dtype == _NS_DTYPE for x in to_concat):
+    is_datetime64 = [x.dtype == _NS_DTYPE for x in to_concat]
+    if all(is_datetime64):
         # work around NumPy 1.6 bug
         new_values = np.concatenate([x.view(np.int64) for x in to_concat],
                                     axis=axis)
         return new_values.view(_NS_DTYPE)
-    else:
-        return np.concatenate(to_concat, axis=axis)
+    elif any(is_datetime64):
+        to_concat = [_to_pydatetime(x) for x in to_concat]
+
+    return np.concatenate(to_concat, axis=axis)
+
+
+def _to_pydatetime(x):
+    if x.dtype == _NS_DTYPE:
+        shape = x.shape
+        x = tslib.ints_to_pydatetime(x.view(np.int64).ravel())
+        x = x.reshape(shape)
+
+    return x
+
 
 def _where_compat(mask, arr1, arr2):
     if arr1.dtype == _NS_DTYPE and arr2.dtype == _NS_DTYPE:

pandas/tseries/tests/test_timeseries.py

@@ -18,6 +18,7 @@
 import pandas.core.datetools as datetools
 import pandas.tseries.offsets as offsets
 import pandas.tseries.frequencies as fmod
+import pandas as pd
 
 from pandas.util.testing import assert_series_equal, assert_almost_equal
 import pandas.util.testing as tm
@@ -1315,7 +1316,6 @@ def test_frame_datetime64_handling_groupby(self):
 
     def test_series_interpolate_intraday(self):
         # #1698
-        import pandas as pd
         index = pd.date_range('1/1/2012', periods=4, freq='12D')
         ts = pd.Series([0, 12, 24, 36], index)
         new_index = index.append(index + pd.DateOffset(days=1)).order()
@@ -1409,6 +1409,21 @@ def f(x):
         s.apply(f)
         DataFrame(s).applymap(f)
 
+    def test_concat_datetime_datetime64_frame(self):
+        # #2624
+        rows = []
+        rows.append([datetime(2010, 1, 1), 1])
+        rows.append([datetime(2010, 1, 2), 'hi'])
+
+        df2_obj = DataFrame.from_records(rows, columns=['date', 'test'])
+
+        ind = date_range(start="2000/1/1", freq="D", periods=10)
+        df1 = DataFrame({'date': ind, 'test':range(10)})
+
+        # it works!
+        pd.concat([df1, df2_obj])
+
+
 def _simple_ts(start, end, freq='D'):
     rng = date_range(start, end, freq=freq)
     return Series(np.random.randn(len(rng)), index=rng)