Merge pull request #10171 from jreback/dt_format

BUG: consistent datetime display format with < ms #10170

Author: jorisvandenbossche

doc/source/whatsnew/v0.17.0.txt

@@ -68,7 +68,7 @@ Bug Fixes
 - Bug in ``NaT`` raises ``AttributeError`` when accessing to ``daysinmonth``, ``dayofweek`` properties. (:issue:`10096`)
 
 - Bug in getting timezone data with ``dateutil`` on various platforms ( :issue:`9059`, :issue:`8639`, :issue:`9663`, :issue:`10121`)
-
+- Bug in display datetimes with mixed frequencies uniformly; display 'ms' datetimes to the proper precision. (:issue:`10170`)
 
 
 - Bug in ``DatetimeIndex`` and ``TimedeltaIndex`` names are lost after timedelta arithmetics ( :issue:`9926`)

pandas/tests/test_format.py

@@ -14,7 +14,7 @@
 from numpy.random import randn
 import numpy as np
 
-from pandas import DataFrame, Series, Index, Timestamp, MultiIndex
+from pandas import DataFrame, Series, Index, Timestamp, MultiIndex, date_range, NaT
 
 import pandas.core.format as fmt
 import pandas.util.testing as tm
@@ -2495,7 +2495,7 @@ def test_to_string(self):
 
     def test_freq_name_separation(self):
         s = Series(np.random.randn(10),
-                   index=pd.date_range('1/1/2000', periods=10), name=0)
+                   index=date_range('1/1/2000', periods=10), name=0)
 
         result = repr(s)
         self.assertTrue('Freq: D, Name: 0' in result)
@@ -2556,7 +2556,6 @@ def test_float_trim_zeros(self):
 
     def test_datetimeindex(self):
 
-        from pandas import date_range, NaT
         index = date_range('20130102',periods=6)
         s = Series(1,index=index)
         result = s.to_string()
@@ -2574,7 +2573,6 @@ def test_datetimeindex(self):
 
     def test_timedelta64(self):
 
-        from pandas import date_range
         from datetime import datetime, timedelta
 
         Series(np.array([1100, 20], dtype='timedelta64[ns]')).to_string()
@@ -3179,6 +3177,44 @@ def test_date_nanos(self):
         result = fmt.Datetime64Formatter(x).get_result()
         self.assertEqual(result[0].strip(), "1970-01-01 00:00:00.000000200")
 
+    def test_dates_display(self):
+
+        # 10170
+        # make sure that we are consistently display date formatting
+        x = Series(date_range('20130101 09:00:00',periods=5,freq='D'))
+        x.iloc[1] = np.nan
+        result = fmt.Datetime64Formatter(x).get_result()
+        self.assertEqual(result[0].strip(), "2013-01-01 09:00:00")
+        self.assertEqual(result[1].strip(), "NaT")
+        self.assertEqual(result[4].strip(), "2013-01-05 09:00:00")
+
+        x = Series(date_range('20130101 09:00:00',periods=5,freq='s'))
+        x.iloc[1] = np.nan
+        result = fmt.Datetime64Formatter(x).get_result()
+        self.assertEqual(result[0].strip(), "2013-01-01 09:00:00")
+        self.assertEqual(result[1].strip(), "NaT")
+        self.assertEqual(result[4].strip(), "2013-01-01 09:00:04")
+
+        x = Series(date_range('20130101 09:00:00',periods=5,freq='ms'))
+        x.iloc[1] = np.nan
+        result = fmt.Datetime64Formatter(x).get_result()
+        self.assertEqual(result[0].strip(), "2013-01-01 09:00:00.000")
+        self.assertEqual(result[1].strip(), "NaT")
+        self.assertEqual(result[4].strip(), "2013-01-01 09:00:00.004")
+
+        x = Series(date_range('20130101 09:00:00',periods=5,freq='us'))
+        x.iloc[1] = np.nan
+        result = fmt.Datetime64Formatter(x).get_result()
+        self.assertEqual(result[0].strip(), "2013-01-01 09:00:00.000000")
+        self.assertEqual(result[1].strip(), "NaT")
+        self.assertEqual(result[4].strip(), "2013-01-01 09:00:00.000004")
+
+        x = Series(date_range('20130101 09:00:00',periods=5,freq='N'))
+        x.iloc[1] = np.nan
+        result = fmt.Datetime64Formatter(x).get_result()
+        self.assertEqual(result[0].strip(), "2013-01-01 09:00:00.000000000")
+        self.assertEqual(result[1].strip(), "NaT")
+        self.assertEqual(result[4].strip(), "2013-01-01 09:00:00.000000004")
 
 class TestNaTFormatting(tm.TestCase):
     def test_repr(self):

pandas/tseries/tests/test_timeseries.py

@@ -791,7 +791,7 @@ def test_series_repr_nat(self):
         series = Series([0, 1000, 2000, iNaT], dtype='M8[ns]')
 
         result = repr(series)
-        expected = ('0          1970-01-01 00:00:00\n'
+        expected = ('0   1970-01-01 00:00:00.000000\n'
                     '1   1970-01-01 00:00:00.000001\n'
                     '2   1970-01-01 00:00:00.000002\n'
                     '3                          NaT\n'

pandas/tslib.pyx

@@ -1418,20 +1418,36 @@ def format_array_from_datetime(ndarray[int64_t] values, object tz=None, object f
     """
     cdef:
         int64_t val, ns, N = len(values)
+        ndarray[int64_t] consider_values
+        bint show_ms = 0, show_us = 0, show_ns = 0, basic_format = 0
         ndarray[object] result = np.empty(N, dtype=object)
         object ts, res
         pandas_datetimestruct dts
 
     if na_rep is None:
        na_rep = 'NaT'
 
+    # if we don't have a format nor tz, then choose
+    # a format based on precision
+    basic_format = format is None and tz is None
+    if basic_format:
+        consider_values = values[values != iNaT]
+        show_ns = (consider_values%1000).any()
+
+        if not show_ns:
+            consider_values //= 1000
+            show_us = (consider_values%1000).any()
+
+            if not show_ms:
+                consider_values //= 1000
+                show_ms = (consider_values%1000).any()
+
     for i in range(N):
-       val = values[i]
+        val = values[i]
 
-       if val == iNaT:
-          result[i] = na_rep
-       else:
-          if format is None and tz is None:
+        if val == iNaT:
+            result[i] = na_rep
+        elif basic_format:
 
             pandas_datetime_to_datetimestruct(val, PANDAS_FR_ns, &dts)
             res = '%d-%.2d-%.2d %.2d:%.2d:%.2d' % (dts.year,
@@ -1441,27 +1457,29 @@ def format_array_from_datetime(ndarray[int64_t] values, object tz=None, object f
                                                    dts.min,
                                                    dts.sec)
 
-            ns = dts.ps / 1000
-
-            if ns != 0:
-               res += '.%.9d' % (ns + 1000 * dts.us)
-            elif dts.us != 0:
-               res += '.%.6d' % dts.us
+            if show_ns:
+                ns = dts.ps / 1000
+                res += '.%.9d' % (ns + 1000 * dts.us)
+            elif show_us:
+                res += '.%.6d' % dts.us
+            elif show_ms:
+                res += '.%.3d' % (dts.us/1000)
 
             result[i] = res
 
-          else:
-             ts = Timestamp(val, tz=tz)
-             if format is None:
-                 result[i] = str(ts)
-             else:
-
-                 # invalid format string
-                 # requires dates > 1900
-                 try:
-                     result[i] = ts.strftime(format)
-                 except ValueError:
-                     result[i] = str(ts)
+        else:
+
+            ts = Timestamp(val, tz=tz)
+            if format is None:
+                result[i] = str(ts)
+            else:
+
+                # invalid format string
+                # requires dates > 1900
+                try:
+                    result[i] = ts.strftime(format)
+                except ValueError:
+                    result[i] = str(ts)
 
     return result