BUG: Rewrite _make_concat_multiindex

closes #20565

Author: toobaz

doc/source/whatsnew/v0.25.0.rst

@@ -183,6 +183,7 @@ Reshaping
 
 - Bug in :func:`merge` when merging by index name would sometimes result in an incorrectly numbered index (:issue:`24212`)
 - :func:`to_records` now accepts dtypes to its `column_dtypes` parameter (:issue:`24895`)
+- Bug in :func:`concat` creating a malformed :class:`MultiIndex` when passed multiple frames indexed by identical :class:`MultiIndex`es (:issue:`20565`)
 -
 
 

pandas/core/reshape/concat.py

@@ -8,12 +8,8 @@
 
 from pandas import DataFrame, Index, MultiIndex, Series, compat
 from pandas.core import common as com
-from pandas.core.arrays.categorical import (
-    _factorize_from_iterable, _factorize_from_iterables)
 from pandas.core.generic import NDFrame
-from pandas.core.index import (
-    _all_indexes_same, _get_consensus_names, _get_objs_combined_axis,
-    ensure_index)
+from pandas.core.index import _get_objs_combined_axis, ensure_index
 import pandas.core.indexes.base as ibase
 from pandas.core.internals import concatenate_block_managers
 
@@ -533,103 +529,62 @@ def _concat_indexes(indexes):
 
 
 def _make_concat_multiindex(indexes, keys, levels=None, names=None):
+    """
+    Produce a MultiIndex which includes concatenated pieces in "indexes",
+    prepended by one or more levels defined by "keys".
 
-    if ((levels is None and isinstance(keys[0], tuple)) or
-            (levels is not None and len(levels) > 1)):
-        zipped = compat.lzip(*keys)
-        if names is None:
-            names = [None] * len(zipped)
-
-        if levels is None:
-            _, levels = _factorize_from_iterables(zipped)
-        else:
-            levels = [ensure_index(x) for x in levels]
-    else:
-        zipped = [keys]
-        if names is None:
-            names = [None]
-
-        if levels is None:
-            levels = [ensure_index(keys)]
-        else:
-            levels = [ensure_index(x) for x in levels]
-
-    if not _all_indexes_same(indexes):
-        codes_list = []
-
-        # things are potentially different sizes, so compute the exact codes
-        # for each level and pass those to MultiIndex.from_arrays
-
-        for hlevel, level in zip(zipped, levels):
-            to_concat = []
-            for key, index in zip(hlevel, indexes):
-                try:
-                    i = level.get_loc(key)
-                except KeyError:
-                    raise ValueError('Key {key!s} not in level {level!s}'
-                                     .format(key=key, level=level))
-
-                to_concat.append(np.repeat(i, len(index)))
-            codes_list.append(np.concatenate(to_concat))
+    Parameters
+    ----------
+    indexes : sequence of Index (or subclass) instances.
+        Pieces of new Index.
+    keys : sequence of labels, same length as "indexes".
+        Labels used to index the pieces in "indexes".
+    levels : list of sequences, default None
+        Used to override the ".levels" in the resulting hierarchical index.
+    names : list, default None
+        Names for the levels in the resulting hierarchical index.
 
-        concat_index = _concat_indexes(indexes)
+    Returns
+    -------
+    concatenated : MultiIndex
 
-        # these go at the end
-        if isinstance(concat_index, MultiIndex):
-            levels.extend(concat_index.levels)
-            codes_list.extend(concat_index.codes)
-        else:
-            codes, categories = _factorize_from_iterable(concat_index)
-            levels.append(categories)
-            codes_list.append(codes)
+    """
 
-        if len(names) == len(levels):
-            names = list(names)
+    orig = _concat_indexes(indexes)
+
+    # Simplest way to create and prepend the keys level(s):
+    keys_chunks = [([key] * len(idx)) for (key, idx) in zip(keys, indexes)]
+    keys_levs = Index([i for l in keys_chunks for i in l],
+                      tupleize_cols=True)
+    tot_df = concat([keys_levs.to_frame().reset_index(drop=True),
+                     orig.to_frame().reset_index(drop=True)], axis=1)
+    temp_names = [None] * keys_levs.nlevels + list(orig.names)
+    result = MultiIndex.from_frame(tot_df, names=temp_names)
+
+    if names is not None:
+        if len(names) == keys_levs.nlevels:
+            # Received only names for keys level(s)
+            result.names = list(names) + list(result.names)[len(names):]
         else:
-            # make sure that all of the passed indices have the same nlevels
-            if not len({idx.nlevels for idx in indexes}) == 1:
-                raise AssertionError("Cannot concat indices that do"
-                                     " not have the same number of levels")
-
-            # also copies
-            names = names + _get_consensus_names(indexes)
-
-        return MultiIndex(levels=levels, codes=codes_list, names=names,
-                          verify_integrity=False)
-
-    new_index = indexes[0]
-    n = len(new_index)
-    kpieces = len(indexes)
-
-    # also copies
-    new_names = list(names)
-    new_levels = list(levels)
-
-    # construct codes
-    new_codes = []
-
-    # do something a bit more speedy
-
-    for hlevel, level in zip(zipped, levels):
-        hlevel = ensure_index(hlevel)
-        mapped = level.get_indexer(hlevel)
-
-        mask = mapped == -1
-        if mask.any():
-            raise ValueError('Values not found in passed level: {hlevel!s}'
-                             .format(hlevel=hlevel[mask]))
-
-        new_codes.append(np.repeat(mapped, n))
-
-    if isinstance(new_index, MultiIndex):
-        new_levels.extend(new_index.levels)
-        new_codes.extend([np.tile(lab, kpieces) for lab in new_index.codes])
-    else:
-        new_levels.append(new_index)
-        new_codes.append(np.tile(np.arange(n), kpieces))
-
-    if len(new_names) < len(new_levels):
-        new_names.extend(new_index.names)
-
-    return MultiIndex(levels=new_levels, codes=new_codes, names=new_names,
-                      verify_integrity=False)
+            # Received names for all levels
+            result.names = names
+
+    if levels is not None:
+        for i, level in enumerate(levels):
+            if level is None:
+                continue
+            cur_lev = result.levels[i]
+            new_lev = Index(level)
+            not_found = np.where(new_lev.get_indexer(cur_lev) == -1)[0]
+
+            if len(not_found):
+                missing = [level[i] for i in not_found]
+                raise ValueError("Values not found in passed level: "
+                                 "{missing!s}"
+                                 .format(missing=missing))
+            cur_val = result.get_level_values(i)
+            result = (result.set_levels(new_lev, level=i)
+                            .set_labels(new_lev.get_indexer_for(cur_val),
+                                        level=i))
+
+    return result

pandas/tests/reshape/test_concat.py

@@ -1303,14 +1303,10 @@ def test_concat_keys_levels_no_overlap(self):
         df2 = DataFrame(np.random.randn(1, 4), index=['b'])
 
         msg = "Values not found in passed level"
-        with pytest.raises(ValueError, match=msg):
-            concat([df, df],
-                   keys=['one', 'two'], levels=[['foo', 'bar', 'baz']])
-
-        msg = "Key one not in level"
-        with pytest.raises(ValueError, match=msg):
-            concat([df, df2],
-                   keys=['one', 'two'], levels=[['foo', 'bar', 'baz']])
+        for other in df, df2:
+            with pytest.raises(ValueError, match=msg):
+                concat([df, other],
+                       keys=['one', 'two'], levels=[['foo', 'bar', 'baz']])
 
     def test_concat_rename_index(self):
         a = DataFrame(np.random.rand(3, 3),
@@ -2436,6 +2432,15 @@ def test_concat_different_extension_dtypes_upcasts(self):
         ], dtype=object)
         tm.assert_series_equal(result, expected)
 
+    def test_concat_repeated_index(self):
+        # GH 20565
+        df = pd.DataFrame(np.random.randn(3, 2),
+                          columns=['A', 'B'], index=['Z1'] * 3)
+
+        result = pd.concat([df, df], keys=['Key1', 'Key2'],
+                           names=['KEY', 'ID'])
+        expected = pd.Index(['Z1'], name='ID')
+        tm.assert_index_equal(result.index.levels[1], expected)
 
 @pytest.mark.parametrize('pdt', [pd.Series, pd.DataFrame, pd.Panel])
 @pytest.mark.parametrize('dt', np.sctypes['float'])