BUG: GroupBy.ngroup dropna=False inconsistency when using Categorical pandas-dev#50100

Added explicit nan check for test case to make it more readable on failure and implemented the fix.

Author: tascillo

pandas/core/groupby/groupby.py

@@ -3256,8 +3256,12 @@ def ngroup(self, ascending: bool = True):
 
             dtype: type
             if self.grouper.has_dropped_na:
-                comp_ids = np.where(comp_ids == -1, np.nan, comp_ids)
-                dtype = np.float64
+                if self.grouper.dropna:
+                    comp_ids = np.where(comp_ids == -1, np.nan, comp_ids)
+                    dtype = np.float64
+                else:
+                    comp_ids = np.where(comp_ids == -1, len(comp_ids) - 1, comp_ids)
+                    dtype = np.int64
             else:
                 dtype = np.int64
 

pandas/tests/groupby/test_missing.py

@@ -1,4 +1,5 @@
 import numpy as np
+import math
 import pytest
 
 import pandas as pd
@@ -163,5 +164,5 @@ def test_categorical_nan_no_dropna():
     ng = g.ngroup()
     result = ng.iloc[0]
     expected = 0
-    assert isinstance(result, int)
+    assert not math.isnan(result)
     assert result == expected