Don't allow wildcard types in constraints

Fixes #12677

Author: odersky

compiler/src/dotty/tools/dotc/core/ConstraintHandling.scala

@@ -97,15 +97,21 @@ trait ConstraintHandling {
         constraint = constraint.replace(param, bound)
         true
       else
+        val dropWildcards = new ApproximatingTypeMap:
+          if !isUpper then variance = -1
+          def apply(t: Type): Type = t match
+            case WildcardType => range(param.underlying.loBound, param.underlying.hiBound)
+            case _ => mapOver(t)
         // Narrow one of the bounds of type parameter `param`
         // If `isUpper` is true, ensure that `param <: `bound`, otherwise ensure
         // that `param >: bound`.
+        val bound1 = dropWildcards(bound)
         val narrowedBounds =
           val saved = homogenizeArgs
           homogenizeArgs = Config.alignArgsInAnd
           try
-            if isUpper then oldBounds.derivedTypeBounds(lo, hi & bound)
-            else oldBounds.derivedTypeBounds(lo | bound, hi)
+            if isUpper then oldBounds.derivedTypeBounds(lo, hi & bound1)
+            else oldBounds.derivedTypeBounds(lo | bound1, hi)
           finally homogenizeArgs = saved
         val c1 = constraint.updateEntry(param, narrowedBounds)
         (c1 eq constraint)

tests/pos/i12677.scala

@@ -0,0 +1,31 @@
+class F[A]
+object F {
+  def apply[A](a: => A) = new F[A]
+}
+
+trait TC[A] { type Out }
+object TC {
+  implicit def tc[A]: TC[A] { type Out = String } = ???
+}
+
+// ====================================================================================
+object Bug {
+  final class CustomHook[A] {
+    def blah(implicit tc: TC[A]): CustomHook[tc.Out] = ???
+  }
+
+  def i: CustomHook[Int] = ???
+  val f = F(i.blah)
+  f: F[CustomHook[String]] // error
+}
+
+// ====================================================================================
+object Workaround {
+  final class CustomHook[A] {
+    def blah[B](implicit tc: TC[A] { type Out = B }): CustomHook[B] = ??? // raise type
+  }
+
+  def i: CustomHook[Int] = ???
+  val f = F(i.blah)
+  f: F[CustomHook[String]] // works
+}