ENH: convert handles up and down sampling

Author: adamklein

pandas/core/generic.py

@@ -3,7 +3,7 @@
 import numpy as np
 
 from pandas.core.common import save, load
-from pandas.core.index import MultiIndex
+from pandas.core.index import MultiIndex, DatetimeIndex
 import pandas.core.datetools as datetools
 
 #-------------------------------------------------------------------------------
@@ -132,25 +132,53 @@ def groupby(self, by=None, axis=0, level=None, as_index=True, sort=True):
         return groupby(self, by, axis=axis, level=level, as_index=as_index,
                        sort=sort)
 
-    def convert(self, rule, how='last', axis=0, as_index=True):
+    def convert(self, rule, method='pad', how='last', axis=0, as_index=True):
         """
-        Convenience method for frequency conversion of timestamped data
+        Convenience method for frequency conversion and resampling of regular
+        time-series data.
+
+        Parameters
+        ----------
+        rule : the offset string or object representing target conversion
+        how : string, method for down- or re-sampling, default 'last'
+        method : string, method for upsampling, default 'pad'
+        axis : int, optional, default 0
+        as_index : see synonymous argument of groupby
         """
         from pandas.core.groupby import Tinterval, translateGrouping
 
         if isinstance(rule, basestring):
             rule = datetools.toOffset(rule)
 
+        idx = self._get_axis(axis)
+        if not isinstance(idx, DatetimeIndex):
+            raise ValueError("Cannot call convert with non-DatetimeIndex")
+
+        if idx.offset is None:
+            raise ValueError("Cannot call convert with non-regular index")
+
         if not isinstance(rule, datetools.DateOffset):
             raise ValueError("Rule not a recognized offset")
 
-        interval = Tinterval(rule, label='right', closed='right')
-        grouped  = self.groupby(interval, axis=axis, as_index=as_index)
+        interval = Tinterval(rule, label='right', closed='right', _obj=self)
 
-        if isinstance(how, basestring):
-            how = translateGrouping(how)
+        currfreq = len(idx)
+        targfreq = len(interval.binner) - 2 # since binner extends endpoints
 
-        return grouped.agg(how)
+        if targfreq <= currfreq:
+            # down- or re-sampling
+            grouped  = self.groupby(interval, axis=axis, as_index=as_index)
+
+            if isinstance(how, basestring):
+                how = translateGrouping(how)
+
+            result = grouped.agg(how)
+        else:
+            # upsampling
+            result = self.reindex(interval.binner[1:-1].view('M8[us]'),
+                                  method=method)
+
+        return result
 
 
     def select(self, crit, axis=0):

pandas/core/groupby.py

@@ -626,7 +626,7 @@ def get_group_levels(self):
 
         return name_list
 
-def generate_bins_generic(values, binner, closed, label):
+def generate_bins_generic(values, binner, closed, label, drop):
     """
     Generate bin edge offsets and bin labels for one array using another array
     which has bin edge values. Both arrays must be sorted.
@@ -690,9 +690,9 @@ def generate_bins_generic(values, binner, closed, label):
         if j >= lenidx:
             break
 
-        # if we've seen some values, mark bin
-        if vc != 0:
-            bins[bc] = j 
+        # if we've seen some values or not ignoring empty bins
+        if vc != 0 or not drop:
+            bins[bc] = j
             bc += 1
             vc = 0
 
@@ -752,20 +752,27 @@ class Tinterval(Grouper, CustomGrouper):
     begin = None
     end = None
     nperiods = None
+    binner = None
 
     def __init__(self, interval='Min', closed='left', label='left',
-                 begin=None, end=None, nperiods=None):
+                 begin=None, end=None, nperiods=None, _obj=None):
         self.offset = interval
         self.closed = closed
         self.label = label
         self.begin = begin
         self.end = end
         self.nperiods = None
 
+        if _obj is not None:
+            self.set_obj(_obj)
+
     def set_obj(self, obj):
         """
         Injects the object we'll act on, which we use to initialize grouper
         """
+        if id(self.obj) == id(obj):
+            return
+
         self.obj = obj
 
         if not isinstance(obj.index, DatetimeIndex):
@@ -778,14 +785,14 @@ def set_obj(self, obj):
             self.binlabels = []
             return
 
-        binner = _generate_time_binner(obj.index, self.offset, self.begin,
-                                       self.end, self.nperiods)
+        self.binner = _generate_time_binner(obj.index, self.offset, self.begin,
+                                            self.end, self.nperiods)
 
-        if isinstance(binner, DatetimeIndex):
-            binner = binner.asi8
+        if isinstance(self.binner, DatetimeIndex):
+            self.binner = self.binner.asi8
 
         # general version, knowing nothing about relative frequencies
-        bins, labels = lib.generate_bins_dt64(index.asi8, binner,
+        bins, labels = lib.generate_bins_dt64(index.asi8, self.binner,
                                               self.closed, self.label)
 
         self.bins = bins
@@ -1767,14 +1774,16 @@ def numpy_groupby(data, labels, axis=0):
 # Helper functions
 
 def translateGrouping(how):
-    if how == 'olhc':
-        return {'open' : lambda arr: arr[0],
-                'low' : lambda arr: arr.min(),
-                'high' : lambda arr: arr.max(),
+    if set(how) == set('ohlc'):
+        return {'open'  : lambda arr: arr[0],
+                'low'   : lambda arr: arr.min(),
+                'high'  : lambda arr: arr.max(),
                 'close' : lambda arr: arr[-1]}
 
-    if how == 'last':
-        return lambda arr: arr[-1]
+    if how in 'last':
+        def picker(arr):
+            return arr[-1] if arr is not None and len(arr) else np.nan
+        return picker
 
     raise ValueError("Unrecognized method: %s" % how)
 

pandas/core/series.py

@@ -599,6 +599,7 @@ def iget_value(self, i):
                 return self[label]
 
     iget = iget_value
+    irow = iget_value
 
     def get_value(self, label):
         """

pandas/tests/test_datetime64.py

@@ -345,12 +345,78 @@ def test_convert_basic(self):
 
         assert_series_equal(result, expect)
 
+        # from daily
+        dti = DatetimeIndex(start=datetime(2005,1,1), 
+                            end=datetime(2005,1,10), offset='D')
+
+        s = Series(rand(len(dti)), dti)
+
+        # to weekly
+        result = s.convert('W') # implicitly @SUN
+
+        self.assertEquals(len(result), 3)
+        self.assert_((result.index.dayofweek == [6,6,6]).all())
+        self.assertEquals(result.irow(0), s['1/2/2005'])
+        self.assertEquals(result.irow(1), s['1/9/2005'])
+        self.assertEquals(result.irow(2), s.irow(-1))
+
+        result = s.convert('W@MON')
+        self.assertEquals(len(result), 2)
+        self.assert_((result.index.dayofweek == [0,0]).all())
+        self.assertEquals(result.irow(0), s['1/3/2005'])
+        self.assertEquals(result.irow(1), s['1/10/2005'])
+
+        result = s.convert('W@TUE')
+        self.assertEquals(len(result), 2)
+        self.assert_((result.index.dayofweek == [1,1]).all())
+        self.assertEquals(result.irow(0), s['1/4/2005'])
+        self.assertEquals(result.irow(1), s['1/10/2005'])
+
+        result = s.convert('W@WED')
+        self.assertEquals(len(result), 2)
+        self.assert_((result.index.dayofweek == [2,2]).all())
+        self.assertEquals(result.irow(0), s['1/5/2005'])
+        self.assertEquals(result.irow(1), s['1/10/2005'])
+
+        result = s.convert('W@THU')
+        self.assertEquals(len(result), 2)
+        self.assert_((result.index.dayofweek == [3,3]).all())
+        self.assertEquals(result.irow(0), s['1/6/2005'])
+        self.assertEquals(result.irow(1), s['1/10/2005'])
+
+        result = s.convert('W@FRI')
+        self.assertEquals(len(result), 2)
+        self.assert_((result.index.dayofweek == [4,4]).all())
+        self.assertEquals(result.irow(0), s['1/7/2005'])
+        self.assertEquals(result.irow(1), s['1/10/2005'])
+
+        # to biz day
+        result = s.convert('B')
+        self.assertEquals(len(result), 6)
+        self.assert_((result.index.dayofweek == [0,1,2,3,4,0]).all())
+        self.assertEquals(result.irow(0), s['1/3/2005'])
+        self.assertEquals(result.irow(1), s['1/4/2005'])
+        self.assertEquals(result.irow(5), s['1/10/2005'])
+
+    def test_convert_upsample(self):
+        # from daily
+        dti = DatetimeIndex(start=datetime(2005,1,1), 
+                            end=datetime(2005,1,10), offset='D')
+
+        s = Series(rand(len(dti)), dti)
+
+        # to minutely, by padding
+        result = s.convert('Min', method='pad')
+        self.assertEquals(len(result), 12961)
+        self.assertEquals(result[0], s[0])
+        self.assertEquals(result[-1], s[-1])
+
     def test_convert_olhc(self):
         s = self.series
 
         grouper = Tinterval(Minute(5), closed='right', label='right')
         expect = s.groupby(grouper).agg(lambda x: x[-1])
-        result = s.convert('5Min', 'olhc')
+        result = s.convert('5Min', how='ohlc')
 
         self.assertEquals(len(result), len(expect))
         self.assertEquals(len(result.columns), 4)