8370481: C2 SuperWord: Long/Integer.compareUnsigned return wrong value in SLP

Reviewed-by: epeter, tonyp

Author: Hamlin Li

src/hotspot/share/opto/subnode.hpp

@@ -331,6 +331,7 @@ struct BoolTest {
   mask negate( ) const { return negate_mask(_test); }
   // Return the negative mask for the given mask, for both signed and unsigned comparison.
   static mask negate_mask(mask btm) { return mask(btm ^ 4); }
+  static mask unsigned_mask(mask btm) { return mask(btm | unsigned_compare); }
   bool is_canonical( ) const { return (_test == BoolTest::ne || _test == BoolTest::lt || _test == BoolTest::le || _test == BoolTest::overflow); }
   bool is_less( )  const { return _test == BoolTest::lt || _test == BoolTest::le; }
   bool is_greater( ) const { return _test == BoolTest::gt || _test == BoolTest::ge; }

src/hotspot/share/opto/superword.cpp

@@ -1698,7 +1698,9 @@ VTransformBoolTest PackSet::get_bool_test(const Node_List* bool_pack) const {
   CmpNode* cmp0 = bol->in(1)->as_Cmp();
   assert(get_pack(cmp0) != nullptr, "Bool must have matching Cmp pack");
 
-  if (cmp0->Opcode() == Op_CmpF || cmp0->Opcode() == Op_CmpD) {
+  switch (cmp0->Opcode()) {
+  case Op_CmpF:
+  case Op_CmpD:
     // If we have a Float or Double comparison, we must be careful with
     // handling NaN's correctly. CmpF and CmpD have a return code, as
     // they are based on the java bytecodes fcmpl/dcmpl:
@@ -1742,7 +1744,24 @@ VTransformBoolTest PackSet::get_bool_test(const Node_List* bool_pack) const {
       mask = bol->_test.negate();
       is_negated = true;
     }
-  }
+    break;
+  case Op_CmpU:
+  case Op_CmpUL:
+    // When we have CmpU->Bool, the mask of the Bool has no unsigned-ness information,
+    // but the mask is implicitly unsigned only because of the CmpU. Since we will replace
+    // the CmpU->Bool with a single VectorMaskCmp, we need to now make the unsigned-ness
+    // explicit.
+    mask = BoolTest::unsigned_mask(mask);
+    break;
+  case Op_CmpI:
+  case Op_CmpL:
+    // The mask of signed int/long scalar comparisons has the same semantics
+    // as the mask for vector elementwise int/long comparison with VectorMaskCmp.
+    break;
+  default:
+    // Other Cmp ops are not expected to get here.
+    ShouldNotReachHere();
+  } // switch
 
   return VTransformBoolTest(mask, is_negated);
 }