Fix #16899: Better handle X instanceOf P where X is T1 | T2

compiler/src/dotty/tools/dotc/transform/TypeTestsCasts.scala

@@ -16,6 +16,8 @@ import util.Spans._
 import reporting._
 import config.Printers.{ transforms => debug }
 
+import patmat.Typ
+
 /** This transform normalizes type tests and type casts,
  *  also replacing type tests with singleton argument type with reference equality check
  *  Any remaining type tests
@@ -51,7 +53,8 @@ object TypeTestsCasts {
    *  6. if `P = T1 | T2` or `P = T1 & T2`, checkable(X, T1) && checkable(X, T2).
    *  7. if `P` is a refinement type, "it's a refinement type"
    *  8. if `P` is a local class which is not statically reachable from the scope where `X` is defined, "it's a local class"
-   *  9. otherwise, ""
+   *  9. if `X` is `T1 | T2`, checkable(T1, P) && checkable(T2, P) or (isCheckDefinitelyFalse(T1, P) && checkable(T2, P)) or (checkable(T1, P) && isCheckDefinitelyFalse(T2, P)).
+   *  10. otherwise, ""
    */
   def whyUncheckable(X: Type, P: Type, span: Span)(using Context): String = atPhase(Phases.refchecksPhase.next) {
     extension (inline s1: String) inline def &&(inline s2: String): String = if s1 == "" then s2 else s1
@@ -129,6 +132,34 @@ object TypeTestsCasts {
 
     }
 
+    /** Whether the check X.isInstanceOf[P] is definitely false? */
+    def isCheckDefinitelyFalse(X: Type, P: Type)(using Context): Boolean = trace(s"isCheckDefinitelyFalse(${X.show}, ${P.show})") {
+      X.dealias match
+      case AndType(x1, x2) =>
+        isCheckDefinitelyFalse(x1, P) || isCheckDefinitelyFalse(x2, P)
+
+      case x =>
+        P.dealias match
+        case AndType(p1, p2) =>
+          isCheckDefinitelyFalse(x, p1) || isCheckDefinitelyFalse(x, p2)
+
+        case p =>
+          val pSpace = Typ(p)
+          val xSpace = Typ(x)
+          if pSpace.canDecompose then
+            val ps = pSpace.decompose.map(_.tp)
+            ps.forall(p => isCheckDefinitelyFalse(x, p))
+          else if xSpace.canDecompose then
+            val xs = xSpace.decompose.map(_.tp)
+            xs.forall(x => isCheckDefinitelyFalse(x, p))
+          else
+            val xClass = effectiveClass(x.widen)
+            val pClass = effectiveClass(p.widen)
+
+            !xClass.derivesFrom(pClass)
+            && (xClass.is(Final) || pClass.is(Final) || !xClass.is(Trait) && !pClass.is(Trait))
+    }
+
     def recur(X: Type, P: Type): String = (X <:< P) ||| (P.dealias match {
       case _: SingletonType     => ""
       case _: TypeProxy
@@ -146,7 +177,19 @@ object TypeTestsCasts {
             //   - T1 <:< T2 | T3
             //   - T1 & T2 <:< T3
             // See TypeComparer#either
-            recur(tp1, P) && recur(tp2, P)
+            val res1 = recur(tp1, P)
+            val res2 = recur(tp2, P)
+
+            if res1.isEmpty && res2.isEmpty then res1
+            else if res2.isEmpty then
+              if isCheckDefinitelyFalse(tp1, P) then res2
+              else i"it cannot be checked against the type $tp1"
+            else if res1.isEmpty then
+              if isCheckDefinitelyFalse(tp2, P) then res1
+              else i"it cannot be checked against the type $tp2"
+            else
+              i"it cannot be checked against the type $tp2"
+
           case _ =>
             // always false test warnings are emitted elsewhere
             X.classSymbol.exists && P.classSymbol.exists &&
@@ -302,8 +345,8 @@ object TypeTestsCasts {
 
         /** Transform isInstanceOf
          *
-         *    expr.isInstanceOf[A | B]  ~~>  expr.isInstanceOf[A] | expr.isInstanceOf[B]
-         *    expr.isInstanceOf[A & B]  ~~>  expr.isInstanceOf[A] & expr.isInstanceOf[B]
+         *    expr.isInstanceOf[A | B]          ~~>  expr.isInstanceOf[A] | expr.isInstanceOf[B]
+         *    expr.isInstanceOf[A & B]          ~~>  expr.isInstanceOf[A] & expr.isInstanceOf[B]
          *    expr.isInstanceOf[Tuple]          ~~>  scala.runtime.Tuples.isInstanceOfTuple(expr)
          *    expr.isInstanceOf[EmptyTuple]     ~~>  scala.runtime.Tuples.isInstanceOfEmptyTuple(expr)
          *    expr.isInstanceOf[NonEmptyTuple]  ~~>  scala.runtime.Tuples.isInstanceOfNonEmptyTuple(expr)

tests/neg-custom-args/isInstanceOf/i5826.scala

@@ -1,6 +1,6 @@
 class Foo {
   def test[A]: List[Int] | A => Int = {
-    case ls: List[Int] => ls.head       // error
+    case ls: List[Int] => ls.head       // ok, List decomposes to Some and None
     case _ => 0
   }
 

tests/pos-special/isInstanceOf/i16899.scala

@@ -0,0 +1,5 @@
+sealed trait Unset
+
+def foo(v: Unset|Option[Int]): Unit = v match
+  case v: Unset => ()
+  case v: Option[Int] => ()