BUG: MultiIndex not dropping nan level and invalid code value

doc/source/whatsnew/v0.25.0.rst

@@ -387,8 +387,8 @@ MultiIndex
 ^^^^^^^^^^
 
 - Bug in which incorrect exception raised by :class:`Timedelta` when testing the membership of :class:`MultiIndex` (:issue:`24570`)
--
--
+- Bug in having code value < -1
+- Bug in not dropping nan level (:issue:`19387`)
 
 I/O
 ^^^

pandas/core/indexes/multi.py

@@ -243,10 +243,19 @@ def __new__(cls, levels=None, codes=None, sortorder=None, names=None,
 
         if verify_integrity:
             result._verify_integrity()
+
+        codes = [cls._reassign_na_codes(*it) for it in zip(levels, codes)]
+        result._set_codes(codes, validate=False)
+
         if _set_identity:
             result._reset_identity()
+
         return result
 
+    @classmethod
+    def _reassign_na_codes(cls, level, code):
+        return [-1 if x == -1 or isna(level[x]) else x for x in code]
+
     def _verify_integrity(self, codes=None, levels=None):
         """
 
@@ -281,6 +290,9 @@ def _verify_integrity(self, codes=None, levels=None):
                                  " level  (%d). NOTE: this index is in an"
                                  " inconsistent state" % (i, level_codes.max(),
                                                           len(level)))
+            if len(level_codes) and level_codes.min() < -1:
+                raise ValueError("On level %d, code value (%d) < -1" %
+                                 (i, level_codes.min()))
             if not level.is_unique:
                 raise ValueError("Level values must be unique: {values} on "
                                  "level {level}".format(

pandas/tests/indexes/multi/test_constructor.py

@@ -67,6 +67,7 @@ def test_constructor_mismatched_codes_levels(idx):
     length_error = (r"On level 0, code max \(3\) >= length of level  \(1\)\."
                     " NOTE: this index is in an inconsistent state")
     label_error = r"Unequal code lengths: \[4, 2\]"
+    code_value_error = (r"On level 0, code value \(-2\) < -1")
 
     # important to check that it's looking at the right thing.
     with pytest.raises(ValueError, match=length_error):
@@ -83,6 +84,23 @@ def test_constructor_mismatched_codes_levels(idx):
     with pytest.raises(ValueError, match=label_error):
         idx.copy().set_codes([[0, 0, 0, 0], [0, 0]])
 
+    # code value smaller than -1
+    with pytest.raises(ValueError, match=code_value_error):
+        MultiIndex(levels=[['a'], ['b']], codes=[[0, -2], [0, 0]])
+
+
+def test_na_levels():
+    tm.assert_index_equal(
+        MultiIndex(levels=[[np.nan, None, pd.NaT, 128, 2]],
+                   codes=[[0, -1, 1, 2, 3, 4]]),
+        MultiIndex(levels=[[np.nan, None, pd.NaT, 128, 2]],
+                   codes=[[-1, -1, -1, -1, 3, 4]]))
+    tm.assert_index_equal(
+        MultiIndex(levels=[[np.nan, 's', pd.NaT, 128, None]],
+                   codes=[[0, -1, 1, 2, 3, 4]]),
+        MultiIndex(levels=[[np.nan, 's', pd.NaT, 128, None]],
+                   codes=[[-1, -1, 1, -1, 3, -1]]))
+
 
 def test_labels_deprecated(idx):
     # GH23752