WIP: high memory in MI

xref #13904

Author: jreback

asv_bench/benchmarks/indexing.py

@@ -88,7 +88,7 @@ def setup(self):
 
     def time_getitem_scalar(self):
         self.ts[self.dt]
-    
+
 
 class DataFrameIndexing(object):
     goal_time = 0.2
@@ -189,6 +189,7 @@ def setup(self):
         self.eps_C = 5
         self.eps_D = 5000
         self.mdt2 = self.mdt.set_index(['A', 'B', 'C', 'D']).sortlevel()
+        self.miint = MultiIndex.from_product([np.arange(1000), np.arange(1000)], names=['one','two'])
 
     def time_series_xs_mi_ix(self):
         self.s.ix[999]
@@ -199,6 +200,10 @@ def time_frame_xs_mi_ix(self):
     def time_multiindex_slicers(self):
         self.mdt2.loc[self.idx[(self.test_A - self.eps_A):(self.test_A + self.eps_A), (self.test_B - self.eps_B):(self.test_B + self.eps_B), (self.test_C - self.eps_C):(self.test_C + self.eps_C), (self.test_D - self.eps_D):(self.test_D + self.eps_D)], :]
 
+    def time_multiindex_get_indexer(self):
+        self.miint.get_indexer(np.array([(0, 10), (0, 11), (0, 12), (0, 13), (0, 14),
+                                         (0, 15), (0, 16),(0, 17), (0, 18), (0, 19)], dtype=object))
+
 
 class PanelIndexing(object):
     goal_time = 0.2

pandas/core/algorithms.py

@@ -1250,7 +1250,7 @@ def take_nd(arr, indexer, axis=0, out=None, fill_value=np.nan, mask_info=None,
         indexer = np.arange(arr.shape[axis], dtype=np.int64)
         dtype, fill_value = arr.dtype, arr.dtype.type()
     else:
-        indexer = _ensure_int64(indexer)
+        indexer = _ensure_int64(indexer, copy=False)
         if not allow_fill:
             dtype, fill_value = arr.dtype, arr.dtype.type()
             mask_info = None, False
@@ -1303,7 +1303,6 @@ def take_nd(arr, indexer, axis=0, out=None, fill_value=np.nan, mask_info=None,
 
     func = _get_take_nd_function(arr.ndim, arr.dtype, out.dtype, axis=axis,
                                  mask_info=mask_info)
-    indexer = _ensure_int64(indexer)
     func(arr, indexer, out, fill_value)
 
     if flip_order:

pandas/hashtable.pxd

@@ -30,6 +30,14 @@ cdef class PyObjectHashTable(HashTable):
     cpdef get_item(self, object val)
     cpdef set_item(self, object key, Py_ssize_t val)
 
+cdef class MultiIndexHashTable(HashTable):
+    cdef:
+        kh_uint64_t *table
+        object mi
+
+    cpdef get_item(self, object val)
+    cpdef set_item(self, object key, Py_ssize_t val)
+
 cdef class StringHashTable(HashTable):
     cdef kh_str_t *table
 

pandas/index.pyx

@@ -284,7 +284,6 @@ cdef class IndexEngine:
         if not self.is_mapping_populated:
 
             values = self._get_index_values()
-
             self.mapping = self._make_hash_table(len(values))
             self.mapping.map_locations(values)
 
@@ -322,7 +321,7 @@ cdef class IndexEngine:
             Py_ssize_t i, j, n, n_t, n_alloc
 
         self._ensure_mapping_populated()
-        values = self._get_index_values()
+        values = np.array(self._get_index_values(), copy=False)
         stargets = set(targets)
         n = len(values)
         n_t = len(targets)
@@ -554,5 +553,36 @@ cdef inline bint _is_utc(object tz):
     return tz is UTC or isinstance(tz, _du_utc)
 
 
+cdef class MultiIndexEngine(IndexEngine):
+
+    def _call_monotonic(self, object mi):
+        return mi.is_lexsorted(), mi.is_monotonic, mi.is_unique
+
+    def get_backfill_indexer(self, other, limit=None):
+        # we coerce to ndarray-of-tuples
+        values = np.array(self._get_index_values())
+        return algos.backfill_object(values, other, limit=limit)
+
+    def get_pad_indexer(self, other, limit=None):
+        # we coerce to ndarray-of-tuples
+        values = np.array(self._get_index_values())
+        return algos.pad_object(values, other, limit=limit)
+
+    cpdef get_loc(self, object val):
+        if is_definitely_invalid_key(val):
+            raise TypeError("'{val}' is an invalid key".format(val=val))
+
+        self._ensure_mapping_populated()
+        if not self.unique:
+            return self._get_loc_duplicates(val)
+
+        try:
+            return self.mapping.get_item(val)
+        except TypeError:
+            raise KeyError(val)
+
+    cdef _make_hash_table(self, n):
+        return _hash.MultiIndexHashTable(n)
+
 # Generated from template.
 include "index_class_helper.pxi"

pandas/indexes/base.py

@@ -2412,7 +2412,6 @@ def _get_fill_indexer_searchsorted(self, target, method, limit=None):
                              'if index and target are monotonic' % method)
 
         side = 'left' if method == 'pad' else 'right'
-        target = np.asarray(target)
 
         # find exact matches first (this simplifies the algorithm)
         indexer = self.get_indexer(target)

pandas/indexes/multi.py

@@ -14,7 +14,6 @@
 from pandas.compat.numpy import function as nv
 from pandas import compat
 
-
 from pandas.types.common import (_ensure_int64,
                                  _ensure_platform_int,
                                  is_object_dtype,
@@ -73,6 +72,7 @@ class MultiIndex(Index):
     _levels = FrozenList()
     _labels = FrozenList()
     _comparables = ['names']
+    _engine_type = _index.MultiIndexEngine
     rename = Index.set_names
 
     def __new__(cls, levels=None, labels=None, sortorder=None, names=None,
@@ -114,7 +114,6 @@ def __new__(cls, levels=None, labels=None, sortorder=None, names=None,
             result._verify_integrity()
         if _set_identity:
             result._reset_identity()
-
         return result
 
     def _verify_integrity(self, labels=None, levels=None):
@@ -619,6 +618,10 @@ def _get_level_number(self, level):
 
     _tuples = None
 
+    @cache_readonly
+    def _engine(self):
+        return self._engine_type(lambda: self, len(self))
+
     @property
     def values(self):
         if self._tuples is not None:
@@ -655,10 +658,59 @@ def _has_complex_internals(self):
         # to disable groupby tricks
         return True
 
+    @cache_readonly
+    def is_monotonic(self):
+
+        # TODO
+        # this is unfortunate we end up tupelizing
+        # just to determine monotonicity :<
+
+        # fast-path
+        if not self.levels[0].is_monotonic:
+            return False
+
+        return Index(self.values).is_monotonic
+
     @cache_readonly
     def is_unique(self):
         return not self.duplicated().any()
 
+    @cache_readonly
+    def _hashed_values(self):
+        """ return a uint64 ndarray of my hashed values """
+        from pandas.tools.hashing import hash_tuples
+        return hash_tuples(self)
+
+    @cache_readonly
+    def _have_mixed_levels(self):
+        """ return a boolean list indicated if we have mixed levels """
+        return ['mixed' in l for l in self._inferred_type_levels]
+
+    @cache_readonly
+    def _inferred_type_levels(self):
+        """ return a list of the inferred types, one for each level """
+        return [i.inferred_type for i in self.levels]
+
+    def _as_valid_indexing_key(self, key):
+        """
+        validate and return our key
+        we need to stringify if we have mixed levels
+
+        this is internal for use for the cython routines
+        """
+        if not isinstance(key, tuple):
+            return key
+
+        if not len(key) == self.nlevels:
+            raise KeyError
+
+        def f(k, stringify):
+            if stringify and not isinstance(k, compat.string_types):
+                k = str(k)
+            return k
+        return tuple([f(k, stringify)
+                      for k, stringify in zip(key, self._have_mixed_levels)])
+
     @deprecate_kwarg('take_last', 'keep', mapping={True: 'last',
                                                    False: 'first'})
     @Appender(base._shared_docs['duplicated'] % ibase._index_doc_kwargs)
@@ -852,7 +904,8 @@ def to_frame(self, index=True):
         from pandas import DataFrame
         result = DataFrame({(name or level): self.get_level_values(level)
                             for name, level in
-                            zip(self.names, range(len(self.levels)))})
+                            zip(self.names, range(len(self.levels)))},
+                           copy=False)
         if index:
             result.index = self
         return result
@@ -1478,29 +1531,41 @@ def get_indexer(self, target, method=None, limit=None, tolerance=None):
         method = missing.clean_reindex_fill_method(method)
         target = _ensure_index(target)
 
-        target_index = target
-        if isinstance(target, MultiIndex):
-            target_index = target._tuple_index
+        # empty indexer
+        if is_list_like(target) and not len(target):
+            return _ensure_platform_int(np.array([]))
+
+        if not isinstance(target, MultiIndex):
+            try:
+                target = MultiIndex.from_tuples(target)
+            except (TypeError, ValueError):
 
-        if not is_object_dtype(target_index.dtype):
-            return np.ones(len(target_index)) * -1
+                # let's instead try with a straight Index
+                if method is None:
+                    return Index(self.values).get_indexer(target,
+                                                          method=method,
+                                                          limit=limit,
+                                                          tolerance=tolerance)
 
         if not self.is_unique:
             raise Exception('Reindexing only valid with uniquely valued Index '
                             'objects')
 
-        self_index = self._tuple_index
-
         if method == 'pad' or method == 'backfill':
             if tolerance is not None:
                 raise NotImplementedError("tolerance not implemented yet "
                                           'for MultiIndex')
-            indexer = self_index._get_fill_indexer(target, method, limit)
+            indexer = self._get_fill_indexer(target, method, limit)
         elif method == 'nearest':
             raise NotImplementedError("method='nearest' not implemented yet "
                                       'for MultiIndex; see GitHub issue 9365')
         else:
-            indexer = self_index._engine.get_indexer(target._values)
+            # we may not compare equally because of hashing if we
+            # don't have the same dtypes
+            if self._inferred_type_levels != target._inferred_type_levels:
+                return Index(self.values).get_indexer(target.values)
+
+            indexer = self._engine.get_indexer(target)
 
         return _ensure_platform_int(indexer)
 
@@ -1567,17 +1632,6 @@ def reindex(self, target, method=None, level=None, limit=None,
 
         return target, indexer
 
-    @cache_readonly
-    def _tuple_index(self):
-        """
-        Convert MultiIndex to an Index of tuples
-
-        Returns
-        -------
-        index : Index
-        """
-        return Index(self._values)
-
     def get_slice_bound(self, label, side, kind):
 
         if not isinstance(label, tuple):
@@ -1824,8 +1878,16 @@ def partial_selection(key, indexer=None):
 
                             key = tuple(self[indexer].tolist()[0])
 
-                        return (self._engine.get_loc(_values_from_object(key)),
-                                None)
+                        try:
+                            return (self._engine.get_loc(
+                                _values_from_object(key)), None)
+                        except ValueError:
+                            # if we hae a very odd MultiIndex,
+                            # e.g. with embedded tuples, this might fail
+                            # TODO: should prob not allow construction of a MI
+                            # like this in the first place
+                            return Index(self.values).get_loc(key)
+
                     else:
                         return partial_selection(key)
                 else:

pandas/io/pytables.py

@@ -3789,9 +3789,9 @@ def read(self, where=None, columns=None, **kwargs):
                 lp = DataFrame(c.data, index=long_index, columns=c.values)
 
                 # need a better algorithm
-                tuple_index = long_index._tuple_index
+                tuple_index = long_index.values
 
-                unique_tuples = lib.fast_unique(tuple_index.values)
+                unique_tuples = lib.fast_unique(tuple_index)
                 unique_tuples = _asarray_tuplesafe(unique_tuples)
 
                 indexer = match(unique_tuples, tuple_index)

pandas/src/algos_common_helper.pxi.in

@@ -567,12 +567,12 @@ def get_dispatch(dtypes):
 
 {{for name, c_type, dtype in get_dispatch(dtypes)}}
 
-cpdef ensure_{{name}}(object arr):
+cpdef ensure_{{name}}(object arr, copy=True):
     if util.is_array(arr):
         if (<ndarray> arr).descr.type_num == NPY_{{c_type}}:
             return arr
         else:
-            return arr.astype(np.{{dtype}})
+            return arr.astype(np.{{dtype}}, copy=copy)
     else:
         return np.array(arr, dtype=np.{{dtype}})