Merge pull request #4684 from jreback/timedelta_fill

ENH: GH3371 support timedelta fillna

Author: jreback

doc/source/release.rst

@@ -67,6 +67,9 @@ pandas 0.13
   - A Series of dtype ``timedelta64[ns]`` can now be divided by another
     ``timedelta64[ns]`` object to yield a ``float64`` dtyped Series. This
     is frequency conversion.
+  - Timedelta64 support ``fillna/ffill/bfill`` with an integer interpreted as seconds,
+    or a ``timedelta`` (:issue:`3371`)
+  - Datetime64 support ``ffill/bfill``
   - Performance improvements with ``__getitem__`` on ``DataFrames`` with
     when the key is a column
   - Support for using a ``DatetimeIndex/PeriodsIndex`` directly in a datelike calculation
@@ -156,6 +159,8 @@ pandas 0.13
   - Remove undocumented/unused ``kind`` keyword argument from ``read_excel``, and ``ExcelFile``. (:issue:`4713`, :issue:`4712`)
   - The ``method`` argument of ``NDFrame.replace()`` is valid again, so that a
     a list can be passed to ``to_replace`` (:issue:`4743`).
+  - provide automatic dtype conversions on _reduce operations (:issue:`3371`)
+  - exclude non-numerics if mixed types with datelike in _reduce operations (:issue:`3371`)
 
 **Internal Refactoring**
 

doc/source/timeseries.rst

@@ -1195,6 +1195,15 @@ issues). ``idxmin, idxmax`` are supported as well.
    df.min().idxmax()
    df.min(axis=1).idxmin()
 
+You can fillna on timedeltas. Integers will be interpreted as seconds. You can
+pass a timedelta to get a particular value.
+
+.. ipython:: python
+
+   y.fillna(0)
+   y.fillna(10)
+   y.fillna(timedelta(days=-1,seconds=5))
+
 .. _timeseries.timedeltas_convert:
 
 Time Deltas & Conversions

doc/source/v0.13.0.txt

@@ -195,6 +195,7 @@ Enhancements
   - NaN handing in get_dummies (:issue:`4446`) with `dummy_na`
 
     .. ipython:: python
+
        # previously, nan was erroneously counted as 2 here
        # now it is not counted at all
        get_dummies([1, 2, np.nan])
@@ -237,10 +238,17 @@ Enhancements
          from pandas import offsets
          td + offsets.Minute(5) + offsets.Milli(5)
 
-    - ``plot(kind='kde')`` now accepts the optional parameters ``bw_method`` and
-      ``ind``, passed to scipy.stats.gaussian_kde() (for scipy >= 0.11.0) to set
-      the bandwidth, and to gkde.evaluate() to specify the indicies at which it
-      is evaluated, respecttively. See scipy docs.
+    - Fillna is now supported for timedeltas
+
+      .. ipython:: python
+
+         td.fillna(0)
+         td.fillna(timedelta(days=1,seconds=5))
+
+  - ``plot(kind='kde')`` now accepts the optional parameters ``bw_method`` and
+    ``ind``, passed to scipy.stats.gaussian_kde() (for scipy >= 0.11.0) to set
+    the bandwidth, and to gkde.evaluate() to specify the indicies at which it
+    is evaluated, respecttively. See scipy docs.
 
 .. _whatsnew_0130.refactoring:
 

pandas/core/common.py

@@ -705,6 +705,54 @@ def diff(arr, n, axis=0):
     return out_arr
 
 
+def _coerce_scalar_to_timedelta_type(r):
+    # kludgy here until we have a timedelta scalar
+    # handle the numpy < 1.7 case
+
+    if is_integer(r):
+        r = timedelta(microseconds=r/1000)
+
+    if _np_version_under1p7:
+        if not isinstance(r, timedelta):
+            raise AssertionError("Invalid type for timedelta scalar: %s" % type(r))
+        if compat.PY3:
+            # convert to microseconds in timedelta64
+            r = np.timedelta64(int(r.total_seconds()*1e9 + r.microseconds*1000))
+        else:
+            return r
+
+    if isinstance(r, timedelta):
+        r = np.timedelta64(r)
+    elif not isinstance(r, np.timedelta64):
+        raise AssertionError("Invalid type for timedelta scalar: %s" % type(r))
+    return r.astype('timedelta64[ns]')
+
+def _coerce_to_dtypes(result, dtypes):
+    """ given a dtypes and a result set, coerce the result elements to the dtypes """
+    if len(result) != len(dtypes):
+        raise AssertionError("_coerce_to_dtypes requires equal len arrays")
+
+    def conv(r,dtype):
+        try:
+            if isnull(r):
+                pass
+            elif dtype == _NS_DTYPE:
+                r = Timestamp(r)
+            elif dtype == _TD_DTYPE:
+                r = _coerce_scalar_to_timedelta_type(r)
+            elif dtype == np.bool_:
+                r = bool(r)
+            elif dtype.kind == 'f':
+                r = float(r)
+            elif dtype.kind == 'i':
+                r = int(r)
+        except:
+            pass
+
+        return r
+
+    return np.array([ conv(r,dtype) for r, dtype in zip(result,dtypes) ])
+
 def _infer_dtype_from_scalar(val):
     """ interpret the dtype from a scalar, upcast floats and ints
         return the new value and the dtype """
@@ -1288,7 +1336,7 @@ def _possibly_cast_to_timedelta(value, coerce=True):
     # coercion compatability
     if coerce == 'compat' and _np_version_under1p7:
 
-        def convert(td, type):
+        def convert(td, dtype):
 
             # we have an array with a non-object dtype
             if hasattr(td,'item'):
@@ -1317,6 +1365,7 @@ def convert(td, type):
         # < 1.7 coercion
         if not is_list_like(value):
             value = np.array([ value ])
+
         dtype = value.dtype
         return np.array([ convert(v,dtype) for v in value ], dtype='m8[ns]')
 

pandas/core/frame.py

@@ -23,7 +23,8 @@
 
 from pandas.core.common import (isnull, notnull, PandasError, _try_sort,
                                 _default_index, _maybe_upcast, _is_sequence,
-                                _infer_dtype_from_scalar, _values_from_object)
+                                _infer_dtype_from_scalar, _values_from_object,
+                                _coerce_to_dtypes, _DATELIKE_DTYPES)
 from pandas.core.generic import NDFrame
 from pandas.core.index import Index, MultiIndex, _ensure_index
 from pandas.core.indexing import (_NDFrameIndexer, _maybe_droplevels,
@@ -4235,11 +4236,24 @@ def _reduce(self, op, axis=0, skipna=True, numeric_only=None,
         axis = self._get_axis_number(axis)
         f = lambda x: op(x, axis=axis, skipna=skipna, **kwds)
         labels = self._get_agg_axis(axis)
+
+        # exclude timedelta/datetime unless we are uniform types
+        if axis == 1 and self._is_mixed_type and len(set(self.dtypes) & _DATELIKE_DTYPES):
+            numeric_only = True
+
         if numeric_only is None:
             try:
                 values = self.values
                 result = f(values)
             except Exception as e:
+
+                # try by-column first
+                if filter_type is None and axis == 0:
+                    try:
+                        return self.apply(f).iloc[0]
+                    except:
+                        pass
+
                 if filter_type is None or filter_type == 'numeric':
                     data = self._get_numeric_data()
                 elif filter_type == 'bool':
@@ -4273,9 +4287,11 @@ def _reduce(self, op, axis=0, skipna=True, numeric_only=None,
                     result = result.astype(np.float64)
                 elif filter_type == 'bool' and notnull(result).all():
                     result = result.astype(np.bool_)
-                # otherwise, accept it
             except (ValueError, TypeError):
-                pass
+
+                # try to coerce to the original dtypes item by item if we can
+                if axis == 0:
+                    result = com._coerce_to_dtypes(result, self.dtypes)
 
         return Series(result, index=labels)
 

pandas/core/generic.py

@@ -20,12 +20,9 @@
                                 _infer_dtype_from_scalar, _maybe_promote,
                                 ABCSeries)
 
-
-
 def is_dictlike(x):
     return isinstance(x, (dict, com.ABCSeries))
 
-
 def _single_replace(self, to_replace, method, inplace, limit):
     orig_dtype = self.dtype
     result = self if inplace else self.copy()
@@ -1906,7 +1903,21 @@ def abs(self):
         abs: type of caller
         """
         obj = np.abs(self)
-        obj = com._possibly_cast_to_timedelta(obj, coerce=False)
+
+        # suprimo numpy 1.6 hacking
+        if com._np_version_under1p7:
+            if self.ndim == 1:
+                if obj.dtype == 'm8[us]':
+                    obj = obj.astype('m8[ns]')
+            elif self.ndim == 2:
+                def f(x):
+                    if x.dtype == 'm8[us]':
+                        x = x.astype('m8[ns]')
+                    return x
+
+                if 'm8[us]' in obj.dtypes.values:
+                    obj = obj.apply(f)
+
         return obj
 
     def groupby(self, by=None, axis=0, level=None, as_index=True, sort=True,