Fix #10108: Keep an isSoft flag in OrTypes

It turned out that it's generally clear whether an OrType should be soft or hard, so
this is a good first step to solve the problem.

Author: odersky

compiler/src/dotty/tools/dotc/core/Definitions.scala

@@ -440,7 +440,7 @@ class Definitions {
   def AnyKindType: TypeRef = AnyKindClass.typeRef
 
   @tu lazy val andType: TypeSymbol = enterBinaryAlias(tpnme.AND, AndType(_, _))
-  @tu lazy val orType: TypeSymbol = enterBinaryAlias(tpnme.OR, OrType(_, _))
+  @tu lazy val orType: TypeSymbol = enterBinaryAlias(tpnme.OR, OrType(_, _, soft = false))
 
   /** Marker method to indicate an argument to a call-by-name parameter.
    *  Created by byNameClosures and elimByName, eliminated by Erasure,

compiler/src/dotty/tools/dotc/core/Hashable.scala

@@ -93,6 +93,9 @@ trait Hashable {
   protected final def doHash(bs: Binders, x1: Int, tp2: Type): Int =
     finishHash(bs, hashing.mix(hashSeed, x1), 1, tp2)
 
+  protected final def doHash(bs: Binders, x1: Int, tp2: Type, tp3: Type): Int =
+    finishHash(bs, hashing.mix(hashSeed, x1), 1, tp2, tp3)
+
   protected final def doHash(bs: Binders, tp1: Type, tp2: Type): Int =
     finishHash(bs, hashSeed, 0, tp1, tp2)
 

compiler/src/dotty/tools/dotc/core/TypeComparer.scala

@@ -614,7 +614,7 @@ class TypeComparer(@constructorOnly initctx: Context) extends ConstraintHandling
             }
         }
         compareTypeLambda
-      case OrType(tp21, tp22) =>
+      case tp2 as OrType(tp21, tp22) =>
         compareAtoms(tp1, tp2) match
           case Some(b) => return b
           case _ =>
@@ -646,12 +646,12 @@ class TypeComparer(@constructorOnly initctx: Context) extends ConstraintHandling
         // solutions. The rewriting delays the point where we have to choose.
         tp21 match {
           case AndType(tp211, tp212) =>
-            return recur(tp1, OrType(tp211, tp22)) && recur(tp1, OrType(tp212, tp22))
+            return recur(tp1, OrType(tp211, tp22, tp2.isSoft)) && recur(tp1, OrType(tp212, tp22, tp2.isSoft))
           case _ =>
         }
         tp22 match {
           case AndType(tp221, tp222) =>
-            return recur(tp1, OrType(tp21, tp221)) && recur(tp1, OrType(tp21, tp222))
+            return recur(tp1, OrType(tp21, tp221, tp2.isSoft)) && recur(tp1, OrType(tp21, tp222, tp2.isSoft))
           case _ =>
         }
         either(recur(tp1, tp21), recur(tp1, tp22)) || fourthTry
@@ -2121,7 +2121,7 @@ class TypeComparer(@constructorOnly initctx: Context) extends ConstraintHandling
       val t2 = distributeOr(tp2, tp1)
       if (t2.exists) t2
       else if (isErased) erasedLub(tp1, tp2)
-      else liftIfHK(tp1, tp2, OrType(_, _), _ | _, _ & _)
+      else liftIfHK(tp1, tp2, OrType(_, _, soft = true), _ | _, _ & _)
     }
   }
 

compiler/src/dotty/tools/dotc/core/Types.scala

@@ -1040,7 +1040,7 @@ object Types {
     def safe_& (that: Type)(using Context): Type = (this, that) match {
       case (TypeBounds(lo1, hi1), TypeBounds(lo2, hi2)) =>
         TypeBounds(
-           OrType.makeHk(lo1.stripLazyRef, lo2.stripLazyRef),
+          OrType.makeHk(lo1.stripLazyRef, lo2.stripLazyRef),
           AndType.makeHk(hi1.stripLazyRef, hi2.stripLazyRef))
       case _ =>
         this & that
@@ -2907,8 +2907,9 @@ object Types {
 
     def derivedAndOrType(tp1: Type, tp2: Type)(using Context) =
       if ((tp1 eq this.tp1) && (tp2 eq this.tp2)) this
-      else if (isAnd) AndType.make(tp1, tp2, checkValid = true)
-      else OrType.make(tp1, tp2)
+      else this match
+        case tp: OrType => OrType.make(tp1, tp2, tp.isSoft)
+        case tp: AndType => AndType.make(tp1, tp2, checkValid = true)
   }
 
   abstract case class AndType(tp1: Type, tp2: Type) extends AndOrType {
@@ -2982,6 +2983,7 @@ object Types {
 
   abstract case class OrType(tp1: Type, tp2: Type) extends AndOrType {
     def isAnd: Boolean = false
+    def isSoft: Boolean
     private var myBaseClassesPeriod: Period = Nowhere
     private var myBaseClasses: List[ClassSymbol] = _
     /** Base classes of are the intersection of the operand base classes. */
@@ -3044,30 +3046,31 @@ object Types {
 
     def derivedOrType(tp1: Type, tp2: Type)(using Context): Type =
       if ((tp1 eq this.tp1) && (tp2 eq this.tp2)) this
-      else OrType.make(tp1, tp2)
+      else OrType.make(tp1, tp2, isSoft)
 
-    override def computeHash(bs: Binders): Int = doHash(bs, tp1, tp2)
+    override def computeHash(bs: Binders): Int =
+      doHash(bs, if isSoft then 0 else 1, tp1, tp2)
 
     override def eql(that: Type): Boolean = that match {
-      case that: OrType => tp1.eq(that.tp1) && tp2.eq(that.tp2)
+      case that: OrType => tp1.eq(that.tp1) && tp2.eq(that.tp2) && isSoft == that.isSoft
       case _ => false
     }
   }
 
-  final class CachedOrType(tp1: Type, tp2: Type) extends OrType(tp1, tp2)
+  final class CachedOrType(tp1: Type, tp2: Type, override val isSoft: Boolean) extends OrType(tp1, tp2)
 
   object OrType {
-    def apply(tp1: Type, tp2: Type)(using Context): OrType = {
+    def apply(tp1: Type, tp2: Type, soft: Boolean)(using Context): OrType = {
       assertUnerased()
-      unique(new CachedOrType(tp1, tp2))
+      unique(new CachedOrType(tp1, tp2, soft))
     }
-    def make(tp1: Type, tp2: Type)(using Context): Type =
+    def make(tp1: Type, tp2: Type, soft: Boolean)(using Context): Type =
       if (tp1 eq tp2) tp1
-      else apply(tp1, tp2)
+      else apply(tp1, tp2, soft)
 
     /** Like `make`, but also supports higher-kinded types as argument */
     def makeHk(tp1: Type, tp2: Type)(using Context): Type =
-      TypeComparer.liftIfHK(tp1, tp2, OrType(_, _), makeHk, _ & _)
+      TypeComparer.liftIfHK(tp1, tp2, OrType(_, _, soft = true), makeHk, _ & _)
   }
 
   /** An extractor object to pattern match against a nullable union.
@@ -3079,7 +3082,7 @@ object Types {
    */
   object OrNull {
     def apply(tp: Type)(using Context) =
-      OrType(tp, defn.NullType)
+      OrType(tp, defn.NullType, soft = false)
     def unapply(tp: Type)(using Context): Option[Type] =
       if (ctx.explicitNulls) {
         val tp1 = tp.stripNull()
@@ -3097,7 +3100,7 @@ object Types {
    */
   object OrUncheckedNull {
     def apply(tp: Type)(using Context) =
-      OrType(tp, defn.UncheckedNullAliasType)
+      OrType(tp, defn.UncheckedNullAliasType, soft = false)
     def unapply(tp: Type)(using Context): Option[Type] =
       if (ctx.explicitNulls) {
         val tp1 = tp.stripUncheckedNull

compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala

@@ -368,7 +368,7 @@ class TreeUnpickler(reader: TastyReader,
             case ANDtype =>
               AndType(readType(), readType())
             case ORtype =>
-              OrType(readType(), readType())
+              OrType(readType(), readType(), soft = false)
             case SUPERtype =>
               SuperType(readType(), readType())
             case MATCHtype =>
@@ -1222,7 +1222,7 @@ class TreeUnpickler(reader: TastyReader,
               val args = until(end)(readTpt())
               val ownType =
                 if (tycon.symbol == defn.andType) AndType(args(0).tpe, args(1).tpe)
-                else if (tycon.symbol == defn.orType) OrType(args(0).tpe, args(1).tpe)
+                else if (tycon.symbol == defn.orType) OrType(args(0).tpe, args(1).tpe, soft = false)
                 else tycon.tpe.safeAppliedTo(args.tpes)
               untpd.AppliedTypeTree(tycon, args).withType(ownType)
             case ANNOTATEDtpt =>

compiler/src/dotty/tools/dotc/quoted/QuoteContextImpl.scala

@@ -1823,7 +1823,7 @@ class QuoteContextImpl private (ctx: Context) extends QuoteContext:
     end OrTypeTypeTest
 
     object OrType extends OrTypeModule:
-      def apply(lhs: TypeRepr, rhs: TypeRepr): OrType = Types.OrType(lhs, rhs)
+      def apply(lhs: TypeRepr, rhs: TypeRepr): OrType = Types.OrType(lhs, rhs, soft = false)
       def unapply(x: OrType): Option[(TypeRepr, TypeRepr)] = Some((x.left, x.right))
     end OrType
 

compiler/src/dotty/tools/dotc/transform/TypeTestsCasts.scala

@@ -70,7 +70,7 @@ object TypeTestsCasts {
         case tref: TypeRef if tref.typeSymbol.isPatternBound =>
           if (variance == 1) tref.info.hiBound
           else if (variance == -1) tref.info.loBound
-          else OrType(defn.AnyType, defn.NothingType)
+          else OrType(defn.AnyType, defn.NothingType, soft = true)
         case _ => mapOver(tp)
       }
     }.apply(tp)

compiler/src/dotty/tools/dotc/transform/patmat/Space.scala

@@ -477,8 +477,8 @@ class SpaceEngine(using Context) extends SpaceLogic {
           else args.map(arg => erase(arg, inArray = false))
         tp.derivedAppliedType(erase(tycon, inArray), args2)
 
-      case OrType(tp1, tp2) =>
-        OrType(erase(tp1, inArray), erase(tp2, inArray))
+      case tp as OrType(tp1, tp2) =>
+        OrType(erase(tp1, inArray), erase(tp2, inArray), tp.isSoft)
 
       case AndType(tp1, tp2) =>
         AndType(erase(tp1, inArray), erase(tp2, inArray))
@@ -870,7 +870,7 @@ class SpaceEngine(using Context) extends SpaceLogic {
       if (ctx.explicitNulls || selTyp.classSymbol.isPrimitiveValueClass)
         project(selTyp)
       else
-        project(OrType(selTyp, constantNullType))
+        project(OrType(selTyp, constantNullType, soft = false))
 
     // in redundancy check, take guard as false in order to soundly approximate
     def projectPrevCases(cases: List[CaseDef]): Space =

compiler/src/dotty/tools/dotc/typer/TypeAssigner.scala

@@ -463,7 +463,7 @@ trait TypeAssigner {
     val ownType =
       if (sameLength(tparams, args))
         if (tycon.symbol == defn.andType) AndType(args(0).tpe, args(1).tpe)
-        else if (tycon.symbol == defn.orType) OrType(args(0).tpe, args(1).tpe)
+        else if (tycon.symbol == defn.orType) OrType(args(0).tpe, args(1).tpe, soft = false)
         else tycon.tpe.appliedTo(args.tpes)
       else wrongNumberOfTypeArgs(tycon.tpe, tparams, args, tree.srcPos)
     tree.withType(ownType)