ENH: DataFrame.{iterrows, to_panel}, MultiIndex.{copy, has_duplicates}. More

testing for the LongPanel removal. Still lots of failing tests

Author: wesm

TODO.rst

@@ -1,9 +1,9 @@
 LongPanel removal
 =================
 
-- level to flex methods
-- level to reindex
-- fast take for items
+- DONE level to flex methods
+- DONE level to reindex
+- ?? fast take for items
 
 
 DONE

pandas/core/frame.py

@@ -375,6 +375,18 @@ def iteritems(self):
         """Iterator over (column, series) pairs"""
         return ((k, self[k]) for k in self.columns)
 
+    def iterrows(self):
+        """
+        Iterate over rows of DataFrame as (index, Series) pairs
+        """
+        from itertools import izip
+        columns = self.columns
+        for k, v in izip(self.index, self.values):
+            s = v.view(Series)
+            s.index = columns
+            s.name = k
+            yield k, s
+
     iterkv = iteritems
     if py3compat.PY3:  # pragma: no cover
         items = iteritems
@@ -687,6 +699,70 @@ def to_sparse(self, fill_value=None, kind='block'):
                                default_kind=kind,
                                default_fill_value=fill_value)
 
+    def to_panel(self):
+        """
+        Transform long (stacked) format (DataFrame) into wide (3D, Panel)
+        format.
+
+        Currently the index of the DataFrame must be a 2-level MultiIndex. This
+        may be generalized later
+
+        Returns
+        -------
+        panel : Panel
+        """
+        from pandas.core.panel import Panel
+
+        wide_shape = (len(self.columns), len(self.index.levels[0]),
+                      len(self.index.levels[1]))
+
+        # only support this kind for now
+        assert(isinstance(self.index, MultiIndex) and
+               len(self.index.levels) == 2)
+
+        major_axis, minor_axis = self.index.levels
+
+        def make_mask(index):
+            """
+            Create observation selection vector using major and minor
+            labels, for converting to wide format.
+            """
+            N, K = index.levshape
+            selector = index.labels[1] + K * index.labels[0]
+            mask = np.zeros(N * K, dtype=bool)
+            mask.put(selector, True)
+            return mask
+
+        def _to_wide_homogeneous():
+            values = np.empty(wide_shape, dtype=self.values.dtype)
+            if not issubclass(values.dtype.type, np.integer):
+                values.fill(np.nan)
+
+            frame_values = self.values
+            for i in xrange(len(self.columns)):
+                values[i].flat[mask] = frame_values[:, i]
+            return Panel(values, self.columns, major_axis, minor_axis)
+
+        def _to_wide_mixed():
+            _, N, K = wide_shape
+            # TODO: make much more efficient
+            data = {}
+            for item in self.columns:
+                item_vals = self[item].values
+                values = np.empty((N, K), dtype=item_vals.dtype)
+                values.flat[mask] = item_vals
+                data[item] = DataFrame(values, index=major_axis,
+                                       columns=minor_axis)
+            return Panel(data, self.columns, major_axis, minor_axis)
+
+        mask = make_mask(self.index)
+        if self._is_mixed_type:
+            return _to_wide_mixed()
+        else:
+            return _to_wide_homogeneous()
+
+    to_wide = deprecate('to_wide', to_panel)
+
     def to_csv(self, path, sep=",", na_rep='', cols=None, header=True,
               index=True, index_label=None, mode='w', nanRep=None):
         """

pandas/core/generic.py

@@ -3,7 +3,7 @@
 import numpy as np
 
 from pandas.core.common import save, load
-from pandas.core.index import _ensure_index
+from pandas.core.index import MultiIndex
 import pandas.core.datetools as datetools
 
 #-------------------------------------------------------------------------------
@@ -118,6 +118,8 @@ def groupby(self, by=None, axis=0, level=None, as_index=True):
         from pandas.core.groupby import groupby
         return groupby(self, by, axis=axis, level=level, as_index=as_index)
 
+    index = None
+
     def truncate(self, before=None, after=None):
         """Function truncate a sorted DataFrame / Series before and/or after
         some particular dates.
@@ -135,8 +137,27 @@ def truncate(self, before=None, after=None):
         """
         before = datetools.to_datetime(before)
         after = datetools.to_datetime(after)
+
+        if before is not None and after is not None:
+            assert(before <= after)
+
         # returns view, want to copy
-        return self.ix[before:after].copy()
+        truncated = self.ix[before:after].copy()
+
+        # slice off chunks of level, adjust labels, a bit of an ugly hack to
+        # get the unit tests to pass
+        index = truncated.index
+        if isinstance(index, MultiIndex):
+            if index is self.index:
+                index = self.index.copy()
+            level = index.levels[0]
+            start, end = level.slice_locs(before, after)
+            index.levels[0] = level[start:end]
+            index.labels[0] = index.labels[0] - start
+
+            truncated.index = index
+
+        return truncated
 
     def select(self, crit, axis=0):
         """

pandas/core/index.py

@@ -981,6 +981,22 @@ def __new__(cls, levels=None, labels=None, sortorder=None, names=None):
 
         return subarr
 
+    def copy(self, order='C'):
+        """
+        Overridden ndarray.copy to copy over attributes
+
+        Returns
+        -------
+        cp : Index
+            Returns view on same base ndarray
+        """
+        cp = self.view(np.ndarray).view(type(self))
+        cp.levels = list(self.levels)
+        cp.labels = list(self.labels)
+        cp.names = list(self.names)
+        cp.sortorder = self.sortorder
+        return cp
+
     @property
     def dtype(self):
         return np.dtype('O')
@@ -1017,6 +1033,23 @@ def _has_complex_internals(self):
         # to disable groupby tricks
         return True
 
+    @property
+    def has_duplicates(self):
+        """
+        Return True if there are no unique groups
+        """
+        # has duplicates
+        shape = [len(lev) for lev in self.levels]
+        group_index = np.zeros(len(self), dtype='i8')
+        for i in xrange(len(shape)):
+            stride = np.prod([x for x in shape[i+1:]], dtype='i8')
+            group_index += self.labels[i] * stride
+
+        if len(np.unique(group_index)) < len(group_index):
+            return True
+
+        return False
+
     def get_level_values(self, level):
         """
         Return vector of label values for requested level, equal to the length
@@ -1179,7 +1212,7 @@ def __getitem__(self, key):
 
             # an optimization
             result = new_tuples.view(MultiIndex)
-            result.levels = self.levels
+            result.levels = list(self.levels)
             result.labels = new_labels
             result.sortorder = sortorder
             result.names = self.names

pandas/core/panel.py

@@ -90,7 +90,7 @@ def f(self, other):
                             'done with scalar values')
 
         return self._combine(other, func)
-
+    f.__name__ = name
     return f
 
 def _panel_arith_method(op, name):
@@ -1160,78 +1160,21 @@ def _get_join_index(self, other, how):
 WidePanel = Panel
 LongPanel = DataFrame
 
-def panel_is_consistent(panel):
-    offset = max(len(panel.major_axis), len(panel.minor_axis))
-    major_labels = panel.major_labels.astype('i8')
-    minor_labels = panel.minor_labels.astype('i8')
-    keys = major_labels * offset + minor_labels
-    unique_keys = np.unique(keys)
-
-    if len(unique_keys) < len(keys):
-        return False
-
-    return True
-
-def long_to_wide(lp):
-    """
-    Transform long (stacked) format into wide format
-
-    Returns
-    -------
-    Panel
-    """
-    assert(lp.consistent)
-    mask = make_mask(lp.index)
-    if lp._data.is_mixed_dtype():
-        return _to_wide_mixed(lp, mask)
-    else:
-        return _to_wide_homogeneous(lp, mask)
-
-def _to_wide_homogeneous(lp, mask):
-    shape = _wide_shape(lp)
-    values = np.empty(shape, dtype=lp.values.dtype)
-
-    if not issubclass(lp.values.dtype.type, np.integer):
-        values.fill(np.nan)
-
-    for i in xrange(len(lp.items)):
-        values[i].flat[mask] = lp.values[:, i]
-
-    return Panel(values, lp.items, lp.major_axis, lp.minor_axis)
-
-def _to_wide_mixed(lp, mask):
-    _, N, K = _wide_shape(lp)
-
-    # TODO: make much more efficient
-
-    data = {}
-    for i, item in enumerate(lp.items):
-        item_vals = lp[item].values
-
-        values = np.empty((N, K), dtype=item_vals.dtype)
-        values.ravel()[mask] = item_vals
-        data[item] = DataFrame(values, index=lp.major_axis,
-                               columns=lp.minor_axis)
-    return Panel.from_dict(data)
-
-def _wide_shape(lp):
-    return (len(lp.columns), len(lp.index.levels[0]), len(lp.index.levels[1]))
-
 def long_swapaxes(frame):
     """
     Swap major and minor axes and reorder values to be grouped by
     minor axis values
 
     Returns
     -------
-    LongPanel (new object)
+    DataFrame (new object)
     """
     return frame.swaplevel(0, 1, axis=0)
 
 
 def long_truncate(lp, before=None, after=None):
     """
-    Slice panel between two major axis values, return complete LongPanel
+    Slice panel between two major axis values, return complete DataFrame
 
     Parameters
     ----------
@@ -1242,7 +1185,7 @@ def long_truncate(lp, before=None, after=None):
 
     Returns
     -------
-    LongPanel
+    DataFrame
     """
     left, right = lp.index.slice_locs(before, after)
     new_index = lp.index.truncate(before, after)
@@ -1260,13 +1203,11 @@ def long_apply(lp, f, axis='major', broadcast=False):
     f : function
         NumPy function to apply to each group
     axis : {'major', 'minor'}
-
     broadcast : boolean
 
     Returns
     -------
-    broadcast=True  -> LongPanel
-    broadcast=False -> DataFrame
+    applied : DataFrame
     """
     try:
         return lp._apply_level(f, axis=axis, broadcast=broadcast)
@@ -1278,8 +1219,8 @@ def long_apply(lp, f, axis='major', broadcast=False):
 
 def make_dummies(frame, item):
     """
-    Use unique values in column of panel to construct LongPanel
-    containing dummy
+    Use unique values in column of panel to construct DataFrame containing
+    dummy variables in the columns (constructed from the unique values)
 
     Parameters
     ----------
@@ -1288,15 +1229,15 @@ def make_dummies(frame, item):
 
     Returns
     -------
-    LongPanel
+    dummies : DataFrame
     """
     from pandas import Factor
     factor = Factor(frame[item].values)
     values = np.eye(len(factor.levels))
     dummy_mat = values.take(factor.labels, axis=0)
     return DataFrame(dummy_mat, columns=factor.levels, index=frame.index)
 
-def make_axis_dummies(frame, axis='minor'):
+def make_axis_dummies(frame, axis='minor', transform=None):
     """
     Construct 1-0 dummy variables corresponding to designated axis
     labels
@@ -1308,19 +1249,27 @@ def make_axis_dummies(frame, axis='minor'):
         Function to apply to axis labels first. For example, to
         get "day of week" dummies in a time series regression you might
         call:
-            panel.get_axis_dummies(axis='major',
-                                   transform=lambda d: d.weekday())
+            make_axis_dummies(panel, axis='major',
+                              transform=lambda d: d.weekday())
     Returns
     -------
-    LongPanel, item names taken from chosen axis
+    dummies : DataFrame
+        Column names taken from chosen axis
     """
     numbers = {
         'major' : 0,
         'minor' : 1
     }
     num = numbers.get(axis, axis)
+
     items = frame.index.levels[num]
     labels = frame.index.labels[num]
+    if transform is not None:
+        mapped_items = items.map(transform)
+        factor = Factor(mapped_items.take(labels))
+        labels = factor.labels
+        items = factor.levels
+
     values = np.eye(len(items), dtype=float)
     values = values.take(labels, axis=0)
 
@@ -1407,16 +1356,5 @@ def _get_distinct_indexes(indexes):
     indexes = sorted(indexes, key=id)
     return [gp.next() for _, gp in groupby(indexes, id)]
 
-def make_mask(index):
-    """
-    Create observation selection vector using major and minor
-    labels, for converting to wide format.
-    """
-    N, K = index.levshape
-    selector = index.labels[1] + K * index.labels[0]
-    mask = np.zeros(N * K, dtype=bool)
-    mask.put(selector, True)
-    return mask
-
 def _monotonic(arr):
     return not (arr[1:] < arr[:-1]).any()