WIP: Skip subtype tests for recursive bounds involving match types

compiler/src/dotty/tools/dotc/core/ConstraintHandling.scala

@@ -109,7 +109,24 @@ trait ConstraintHandling[AbstractContext] {
         (c1 eq constraint) || {
           constraint = c1
           val TypeBounds(lo, hi) = constraint.entry(param)
-          isSubType(lo, hi)
+          hi match {
+            case AppliedType(tr: TypeRef, targs) =>
+              val recursiveBound: Boolean =
+                targs.exists {
+                  case lr: LazyRef => lr.ref == param
+                  case _ => false
+                }
+              if(!recursiveBound) isSubType(lo, hi)
+              else
+                tr.underlying match {
+                  case TypeBounds(trlo, trhi) if trlo.isMatch || trhi.isMatch =>
+                    true
+                  case _ =>
+                    isSubType(lo, hi)
+                }
+            case _ =>
+              isSubType(lo, hi)
+          }
         }
       }
     }

compiler/test/dotc/pos-from-tasty.blacklist

@@ -20,3 +20,5 @@ default-super.scala
 i3050.scala
 i4006b.scala
 i4006c.scala
+
+i5535.scala

compiler/test/dotc/pos-test-pickling.blacklist

@@ -10,3 +10,4 @@ t3249
 t3486
 t3612.scala
 typelevel0.scala
+i5535.scala

tests/neg/i5535.scala

@@ -0,0 +1,15 @@
+object Test2 {
+  // The identity on types of the form N | P[a, P[b, ... P[z, N]]]
+  type Nested[X, P[_, _], N] = X match {
+    case N => N
+    case P[a, b] => P[a, Nested[b, P, N]]
+  }
+
+  // Type bound:
+  //   Recursion limit exceeded.
+  //   Maybe there is an illegal cyclic reference?
+  def p1[T <: Nested[T, Tuple2, Unit]](t: T): T = t
+  //p1(())
+  p1((23, 24))            // error
+  p1(("foo", (23, 24)))   // error
+}

tests/pos/i5535.scala

@@ -0,0 +1,38 @@
+object Test1 {
+  // The identity on Unit
+  type IdUnit[X] = X match {
+    case Unit => Unit
+  }
+
+  // Type bound:
+  //   Recursion limit exceeded.
+  //   Maybe there is an illegal cyclic reference?
+  def p1[T <: IdUnit[T]](t: T): T = t
+  p1(())
+
+  // Type constraint: OK
+  def p2[T](t: T)(implicit ev: T <:< IdUnit[T]): T = t
+  p2(())
+}
+
+object Test2 {
+  // The identity on types of the form N | P[a, P[b, ... P[z, N]]]
+  type Nested[X, P[_, _], N] = X match {
+    case N => N
+    case P[a, b] => P[a, Nested[b, P, N]]
+  }
+
+  // Type bound:
+  //   Recursion limit exceeded.
+  //   Maybe there is an illegal cyclic reference?
+  def p1[T <: Nested[T, Tuple2, Unit]](t: T): T = t
+  p1(())
+  p1((23, ()))
+  p1(("foo", (23, ())))
+
+  // Type constraint: OK
+  def p2[T](t: T)(erased implicit ev: T <:< Nested[T, Tuple2, Unit]): T = t
+  p2(())
+  p2((23, ()))
+  p2(("foo", (23, ())))
+}