wide table support

Author: hhuuggoo

pandas/io/pytables.py

@@ -11,8 +11,11 @@
 import itertools
 import warnings
 import os
+import cPickle as pickle
 
 import numpy as np
+from tables.exceptions import NoSuchNodeError
+
 from pandas import (Series, TimeSeries, DataFrame, Panel, Panel4D, Index,
                     MultiIndex, Int64Index, Timestamp)
 from pandas.sparse.api import SparseSeries, SparseDataFrame, SparsePanel
@@ -44,7 +47,7 @@
 from distutils.version import LooseVersion
 
 # versioning attribute
-_version = '0.15.2'
+_version = '0.17.0'
 
 # PY3 encoding if we don't specify
 _default_encoding = 'UTF-8'
@@ -1394,7 +1397,7 @@ def __init__(self, values=None, kind=None, typ=None, cname=None,
         self.freq = freq
         self.tz = tz
         self.index_name = index_name
-        self.table = None
+        self._handler = None
         self.meta = None
         self.metadata = None
 
@@ -1425,8 +1428,12 @@ def set_pos(self, pos):
             self.typ._v_pos = pos
         return self
 
-    def set_table(self, table):
-        self.table = table
+    @property
+    def table(self):
+        return self._handler.table
+
+    def set_handler(self, handler):
+        self._handler = handler
         return self
 
     def __unicode__(self):
@@ -1463,7 +1470,7 @@ def infer(self, handler):
         """infer this column from the table: create and return a new object"""
         table = handler.table
         new_self = self.copy()
-        new_self.set_table(table)
+        new_self.set_handler(handler)
         new_self.get_attr()
         new_self.read_metadata(handler)
         return new_self
@@ -1536,7 +1543,7 @@ def maybe_set_size(self, min_itemsize=None, **kwargs):
                 ).StringCol(itemsize=min_itemsize, pos=self.pos)
 
     def validate_and_set(self, handler, append, **kwargs):
-        self.set_table(handler.table)
+        self.set_handler(handler)
         self.validate_col()
         self.validate_attr(append)
         self.validate_metadata(handler)
@@ -2060,16 +2067,23 @@ def convert(self, values, nan_rep, encoding):
 
         return self
 
+    def _set_carray_attr(self, key, val):
+        self._handler._set_carray_attr(key, val)
+
+    def _get_carray_attr(self, key):
+        return self._handler._get_carray_attr(key)
+
     def get_attr(self):
         """ get the data for this colummn """
-        self.values = getattr(self.attrs, self.kind_attr, None)
+        self.values = self._get_carray_attr(self.kind_attr)
         self.dtype = getattr(self.attrs, self.dtype_attr, None)
         self.meta = getattr(self.attrs, self.meta_attr, None)
         self.set_kind()
 
+
     def set_attr(self):
         """ set the data for this colummn """
-        setattr(self.attrs, self.kind_attr, self.values)
+        self._set_carray_attr(self.kind_attr, self.values)
         setattr(self.attrs, self.meta_attr, self.meta)
         if self.dtype is not None:
             setattr(self.attrs, self.dtype_attr, self.dtype)
@@ -3026,13 +3040,31 @@ def set_info(self):
         """ update our table index info """
         self.attrs.info = self.info
 
+    def _get_carray_attr(self, name):
+        if 'attrs' in self.group._v_groups:
+            attr_group = self.group._v_groups['attrs']
+            try:
+                data = self._handle.get_node(attr_group, name)[:].tostring()
+                data = pickle.loads(data)
+                return data
+            except NoSuchNodeError:
+                return None
+        return getattr(self.attrs, name, None)
+
+    def _set_carray_attr(self, name, val):
+        if 'attrs' not in self.group._v_groups:
+            self._handle.create_group(self.group, 'attrs')
+        attr_group = self.group._v_groups['attrs']
+        data = np.fromstring(pickle.dumps(val), dtype='S1')
+        self._handle.create_carray(attr_group, name, obj=data)
+
     def set_attrs(self):
         """ set our table type & indexables """
         self.attrs.table_type = str(self.table_type)
-        self.attrs.index_cols = self.index_cols()
-        self.attrs.values_cols = self.values_cols()
-        self.attrs.non_index_axes = self.non_index_axes
-        self.attrs.data_columns = self.data_columns
+        self._set_carray_attr('index_cols', self.index_cols())
+        self._set_carray_attr('values_cols', self.values_cols())
+        self._set_carray_attr('non_index_axes', self.non_index_axes)
+        self._set_carray_attr('data_columns', self.data_columns)
         self.attrs.nan_rep = self.nan_rep
         self.attrs.encoding = self.encoding
         self.attrs.levels = self.levels
@@ -3041,10 +3073,9 @@ def set_attrs(self):
 
     def get_attrs(self):
         """ retrieve our attributes """
-        self.non_index_axes = getattr(
-            self.attrs, 'non_index_axes', None) or []
-        self.data_columns = getattr(
-            self.attrs, 'data_columns', None) or []
+
+        self.non_index_axes = self._get_carray_attr('non_index_axes') or []
+        self.data_columns = self._get_carray_attr('data_columns') or []
         self.info = getattr(
             self.attrs, 'info', None) or dict()
         self.nan_rep = getattr(self.attrs, 'nan_rep', None)
@@ -3098,9 +3129,10 @@ def indexables(self):
             self._indexables = []
 
             # index columns
+            index_cols = self._get_carray_attr('index_cols') or []
             self._indexables.extend([
                 IndexCol(name=name, axis=axis, pos=i)
-                for i, (axis, name) in enumerate(self.attrs.index_cols)
+                for i, (axis, name) in enumerate(index_cols)
             ])
 
             # values columns
@@ -3113,9 +3145,10 @@ def f(i, c):
                     klass = DataIndexableCol
                 return klass.create_for_block(i=i, name=c, pos=base_pos + i,
                                               version=self.version)
-
+            values_cols = self._get_carray_attr('values_cols') or []
             self._indexables.extend(
-                [f(i, c) for i, c in enumerate(self.attrs.values_cols)])
+                [f(i, c) for i, c in enumerate(values_cols)
+             ])
 
         return self._indexables