Special handling of experimental.captureChecking import (#17427)

The question is, how do we introduce capture checking safely. There are
two conflicting requirements:

 - Capture checking should not leak into standard Scala. It should be 
   meaningful only in code that has capture checking explicitly enabled.
- We need to be able to build up libraries that express capture
information
   and that can be consumed from other code. This needs to start with
   the standard library itself.

So far, everything related to capture checking was marked experimental.
This means
all code that refers to a capture checking abstraction in any way
whatsoever needs
to be declared experimental. That clearly does not work for the new use
cases.

But fortunately, capture checking has some properties that enable a
different scheme.
Specifically, a file compiled under capture checking looks like a
completely normal
component (both Tasty and binary) to any other file that uses it and
that is not compiled
with captureChecking. Only when the consumer is also compiled with
capture checking,
the capturing types of the original file will be revealed. The same
holds for binaries.
Capture checking has no effect at all on the binaries that get generated
and all types
and annotations needed for capture checking are erased.

This allows the following more flexible scheme:

- We can turn capture checking on with a setting or language import in
any source file.
   The sources do not have to be @experimental.
- If capture checking is turned on, a number of annotations and other
symbols that
are needed for capture checking and are normally experimental are also
made available.

The important property is that capture checking in one component cannot
poison other normal components. Like
@experimental itself, the whole scheme is transitive. 

With the new scheme we do not need a special exemption for the dotty
package anymore, so the code
implementing the exception is dropped.

Author: odersky

compiler/src/dotty/tools/dotc/config/Feature.scala

@@ -136,16 +136,21 @@ object Feature:
     if !isExperimentalEnabled then
       report.error(em"Experimental $which may only be used with a nightly or snapshot version of the compiler$note", srcPos)
 
+  private def ccException(sym: Symbol)(using Context): Boolean =
+    ccEnabled && defn.ccExperimental.contains(sym)
+
   def checkExperimentalDef(sym: Symbol, srcPos: SrcPos)(using Context) =
     if !isExperimentalEnabled then
-      val symMsg =
-        if sym.hasAnnotation(defn.ExperimentalAnnot) then
-          i"$sym is marked @experimental"
-        else if sym.owner.hasAnnotation(defn.ExperimentalAnnot) then
-          i"${sym.owner} is marked @experimental"
-        else
-          i"$sym inherits @experimental"
-      report.error(em"$symMsg and therefore may only be used in an experimental scope.", srcPos)
+      val experimentalSym =
+        if sym.hasAnnotation(defn.ExperimentalAnnot) then sym
+        else if sym.owner.hasAnnotation(defn.ExperimentalAnnot) then sym.owner
+        else NoSymbol
+      if !ccException(experimentalSym) then
+        val symMsg =
+          if experimentalSym.exists
+          then i"$experimentalSym is marked @experimental"
+          else i"$sym inherits @experimental"
+        report.error(em"$symMsg and therefore may only be used in an experimental scope.", srcPos)
 
   /** Check that experimental compiler options are only set for snapshot or nightly compiler versions. */
   def checkExperimentalSettings(using Context): Unit =

compiler/src/dotty/tools/dotc/core/Definitions.scala

@@ -976,6 +976,8 @@ class Definitions {
     @tu lazy val Caps_unsafeBoxFunArg: Symbol = CapsUnsafeModule.requiredMethod("unsafeBoxFunArg")
     @tu lazy val Caps_SealedAnnot: ClassSymbol = requiredClass("scala.caps.Sealed")
 
+  @tu lazy val PureClass: Symbol = requiredClass("scala.Pure")
+
   // Annotation base classes
   @tu lazy val AnnotationClass: ClassSymbol = requiredClass("scala.annotation.Annotation")
   @tu lazy val StaticAnnotationClass: ClassSymbol = requiredClass("scala.annotation.StaticAnnotation")
@@ -1944,6 +1946,14 @@ class Definitions {
       case Some(pkgs) => pkgs.contains(sym.owner)
       case none => false
 
+  /** Experimental definitions that can nevertheless be accessed from a stable
+   *  compiler if capture checking is enabled.
+   */
+  @tu lazy val ccExperimental: Set[Symbol] = Set(
+    CapsModule, CapsModule.moduleClass, PureClass,
+    CapabilityAnnot, RequiresCapabilityAnnot,
+    RetainsAnnot, RetainsByNameAnnot, WithPureFunsAnnot)
+
   // ----- primitive value class machinery ------------------------------------------
 
   class PerRun[T](generate: Context ?=> T) {
@@ -2041,15 +2051,17 @@ class Definitions {
   def isValueSubClass(sym1: Symbol, sym2: Symbol): Boolean =
     valueTypeEnc(sym2.asClass.name) % valueTypeEnc(sym1.asClass.name) == 0
 
-  @tu lazy val specialErasure: SimpleIdentityMap[Symbol, ClassSymbol] =
-    SimpleIdentityMap.empty[Symbol]
-      .updated(AnyClass, ObjectClass)
-      .updated(MatchableClass, ObjectClass)
-      .updated(AnyValClass, ObjectClass)
-      .updated(SingletonClass, ObjectClass)
-      .updated(TupleClass, ProductClass)
-      .updated(NonEmptyTupleClass, ProductClass)
-      .updated(PairClass, ObjectClass)
+  @tu lazy val specialErasure: collection.Map[Symbol, ClassSymbol] =
+    val m = mutable.Map[Symbol, ClassSymbol]()
+    m(AnyClass) = ObjectClass
+    m(MatchableClass) = ObjectClass
+    m(PureClass) = ObjectClass
+    m(AnyValClass) = ObjectClass
+    m(SingletonClass) = ObjectClass
+    m(TupleClass) = ProductClass
+    m(NonEmptyTupleClass) = ProductClass
+    m(PairClass) = ObjectClass
+    m
 
   // ----- Initialization ---------------------------------------------------
 

compiler/src/dotty/tools/dotc/core/TypeErasure.scala

@@ -410,7 +410,7 @@ object TypeErasure {
             val candidates = takeUntil(tp2superclasses)(!_.is(Trait))
 
             // Candidates st "no other common superclass or trait derives from S"
-            // Also, drop `PairClass` since it is not valid after erasue
+            // Also, drop `PairClass` since it is not valid after erasure
             val minimums = candidates.filter { cand =>
               cand != defn.PairClass
               && candidates.forall(x => !x.derivesFrom(cand) || x.eq(cand))

compiler/src/dotty/tools/dotc/transform/Erasure.scala

@@ -190,12 +190,14 @@ class Erasure extends Phase with DenotTransformer {
   def assertErased(tp: Type, tree: tpd.Tree = tpd.EmptyTree)(using Context): Unit = {
     def isAllowed(cls: Symbol, sourceName: String) =
       tp.typeSymbol == cls && ctx.compilationUnit.source.file.name == sourceName
-    assert(isErasedType(tp) ||
-           isAllowed(defn.ArrayClass, "Array.scala") ||
-           isAllowed(defn.TupleClass, "Tuple.scala") ||
-           isAllowed(defn.NonEmptyTupleClass, "Tuple.scala") ||
-           isAllowed(defn.PairClass, "Tuple.scala"),
-        i"The type $tp - ${tp.toString} of class ${tp.getClass} of tree $tree : ${tree.tpe} / ${tree.getClass} is illegal after erasure, phase = ${ctx.phase.prev}")
+    assert(
+      isErasedType(tp)
+      || isAllowed(defn.ArrayClass, "Array.scala")
+      || isAllowed(defn.TupleClass, "Tuple.scala")
+      || isAllowed(defn.NonEmptyTupleClass, "Tuple.scala")
+      || isAllowed(defn.PairClass, "Tuple.scala")
+      || isAllowed(defn.PureClass, "Pure.scala"),
+      i"The type $tp - ${tp.toString} of class ${tp.getClass} of tree $tree : ${tree.tpe} / ${tree.getClass} is illegal after erasure, phase = ${ctx.phase.prev}")
   }
 }
 

compiler/src/dotty/tools/dotc/transform/SymUtils.scala

@@ -370,8 +370,6 @@ object SymUtils:
       self.hasAnnotation(defn.ExperimentalAnnot)
       || isDefaultArgumentOfExperimentalMethod
       || (!self.is(Package) && self.owner.isInExperimentalScope)
-      || self.topLevelClass.ownersIterator.exists(p =>
-          p.is(Package) && p.owner.isRoot && p.name == tpnme.dotty)
 
     /** The declared self type of this class, as seen from `site`, stripping
     *  all refinements for opaque types.

compiler/src/dotty/tools/dotc/typer/Checking.scala

@@ -784,7 +784,9 @@ object Checking {
     for case imp @ Import(qual, selectors) <- trees do
       def isAllowedImport(sel: untpd.ImportSelector) =
         val name = Feature.experimental(sel.name)
-        name == Feature.scala2macros || name == Feature.erasedDefinitions
+        name == Feature.scala2macros
+        || name == Feature.erasedDefinitions
+        || name == Feature.captureChecking
 
       languageImport(qual) match
         case Some(nme.experimental)

library/src/scala/Pure.scala

@@ -0,0 +1,9 @@
+package scala
+import annotation.experimental
+
+/** A marker trait that declares that all inheriting classes are "pure" in the
+ *  sense that their values retain no capabilities including capabilities needed
+ *  to perform effects. This has formal meaning only under capture checking.
+ */
+@experimental trait Pure:
+  this: Pure =>

library/src/scala/caps.scala

@@ -40,9 +40,3 @@ import annotation.experimental
    */
   @deprecated("The Sealed annotation should not be directly used in source code.\nUse the `sealed` modifier on type parameters instead.")
   class Sealed extends annotation.Annotation
-
-  /** Mixing in this trait forces a trait or class to be pure, i.e.
-   *  have no capabilities retained in its self type.
-   */
-  trait Pure:
-    this: Pure =>

tests/neg-custom-args/captures/selftype.scala

@@ -1,4 +1,4 @@
-@annotation.experimental class C(x: () => Unit) extends caps.Pure // error
+@annotation.experimental class C(x: () => Unit) extends Pure // error
 
-@annotation.experimental class D(@annotation.constructorOnly x: () => Unit) extends caps.Pure // ok
+@annotation.experimental class D(@annotation.constructorOnly x: () => Unit) extends Pure // ok
 

tests/pos-custom-args/no-experimental/cc-experimental.scala

@@ -0,0 +1,11 @@
+package scala.runtime
+
+import language.experimental.captureChecking
+
+object test:
+  type T = Pure
+
+class Foo extends Object, Pure:
+  val x: Pure = ???
+  def foo() = ()
+

tests/pos-custom-args/no-experimental/dotty-experimental.scala

@@ -1,4 +1,4 @@
-package dotty.tools
+import language.experimental.captureChecking
 object test {
 
   val x = caps.cap

tests/pos-with-compiler-cc/backend/jvm/BCodeHelpers.scala

@@ -209,7 +209,7 @@ trait BCodeHelpers extends BCodeIdiomatic with BytecodeWriters {
     }
   } // end of trait BCPickles
 
-  trait BCInnerClassGen extends caps.Pure {
+  trait BCInnerClassGen extends Pure {
 
     def debugLevel = 3 // 0 -> no debug info; 1-> filename; 2-> lines; 3-> varnames
 

tests/pos-with-compiler-cc/backend/jvm/BCodeIdiomatic.scala

@@ -17,7 +17,7 @@ import dotty.tools.dotc.report
  *  @version 1.0
  *
  */
-trait BCodeIdiomatic extends caps.Pure {
+trait BCodeIdiomatic extends Pure {
   val int: DottyBackendInterface
   final lazy val bTypes = new BTypesFromSymbols[int.type](int)
 

tests/pos-with-compiler-cc/backend/jvm/BTypes.scala

@@ -14,7 +14,7 @@ import scala.tools.asm
  * This representation is immutable and independent of the compiler data structures, hence it can
  * be queried by concurrent threads.
  */
-abstract class BTypes extends caps.Pure {
+abstract class BTypes extends Pure {
 
   val int: DottyBackendInterface
   import int.given
@@ -47,7 +47,7 @@ abstract class BTypes extends caps.Pure {
    * A BType is either a primitve type, a ClassBType, an ArrayBType of one of these, or a MethodType
    * referring to BTypes.
    */
-  /*sealed*/ trait BType extends caps.Pure { // Not sealed for now due to SI-8546
+  /*sealed*/ trait BType extends Pure { // Not sealed for now due to SI-8546
     final override def toString: String = this match {
       case UNIT   => "V"
       case BOOL   => "Z"

tests/pos-with-compiler-cc/backend/sjs/ScopedVar.scala

@@ -1,6 +1,6 @@
 package dotty.tools.backend.sjs
 
-class ScopedVar[A](init: A) extends caps.Pure {
+class ScopedVar[A](init: A) extends Pure {
   import ScopedVar.Assignment
 
   private[ScopedVar] var value = init

tests/pos-with-compiler-cc/dotc/ast/Positioned.scala

@@ -15,7 +15,7 @@ import annotation.internal.sharable
 
 /** A base class for things that have positions (currently: modifiers and trees)
  */
-abstract class Positioned(implicit @constructorOnly src: SourceFile) extends SrcPos, Product, Cloneable, caps.Pure {
+abstract class Positioned(implicit @constructorOnly src: SourceFile) extends SrcPos, Product, Cloneable, Pure {
   import Positioned.{ids, nextId, debugId}
 
   private var mySpan: Span = _

tests/pos-with-compiler-cc/dotc/cc/CaptureSet.scala

@@ -39,7 +39,7 @@ import annotation.retains
  *  if the mapped function is either a bijection or if it is idempotent
  *  on capture references (c.f. doc comment on `map` below).
  */
-sealed abstract class CaptureSet extends Showable, caps.Pure:
+sealed abstract class CaptureSet extends Showable, Pure:
   import CaptureSet.*
 
   /** The elements of this capture set. For capture variables,

tests/pos-with-compiler-cc/dotc/core/Annotations.scala

@@ -17,7 +17,7 @@ object Annotations {
     if (tree.symbol.isConstructor) tree.symbol.owner
     else tree.tpe.typeSymbol
 
-  abstract class Annotation extends Showable, caps.Pure {
+  abstract class Annotation extends Showable, Pure {
 
     def tree(using Context): Tree
 

tests/pos-with-compiler-cc/dotc/core/CheckRealizable.scala

@@ -13,7 +13,7 @@ import annotation.constructorOnly
 /** Realizability status */
 object CheckRealizable {
 
-  sealed abstract class Realizability(val msg: String) extends caps.Pure {
+  sealed abstract class Realizability(val msg: String) extends Pure {
     def andAlso(other: => Realizability): Realizability =
       if (this == Realizable) other else this
     def mapError(f: Realizability -> Context ?-> Realizability)(using Context): Realizability =

tests/pos-with-compiler-cc/dotc/core/Denotations.scala

@@ -76,7 +76,7 @@ object Denotations {
   /** A PreDenotation represents a group of single denotations or a single multi-denotation
    *  It is used as an optimization to avoid forming MultiDenotations too eagerly.
    */
-  abstract class PreDenotation extends caps.Pure {
+  abstract class PreDenotation extends Pure {
 
     /** A denotation in the group exists */
     def exists: Boolean

tests/pos-with-compiler-cc/dotc/core/NameKinds.scala

@@ -23,14 +23,14 @@ object NameKinds {
   @sharable private val uniqueNameKinds = util.HashMap[String, UniqueNameKind]()
 
   /** A class for the info stored in a derived name */
-  abstract class NameInfo extends caps.Pure {
+  abstract class NameInfo extends Pure {
     def kind: NameKind
     def mkString(underlying: TermName): String
     def map(f: SimpleName => SimpleName): NameInfo = this
   }
 
   /** An abstract base class of classes that define the kind of a derived name info */
-  abstract class NameKind(val tag: Int) extends caps.Pure { self =>
+  abstract class NameKind(val tag: Int) extends Pure { self =>
 
     /** The info class defined by this kind */
     type ThisInfo <: Info

tests/pos-with-compiler-cc/dotc/core/Names.scala

@@ -32,7 +32,7 @@ object Names {
    *  in a name table. A derived term name adds a tag, and possibly a number
    *  or a further simple name to some other name.
    */
-  abstract class Name extends Designator, Showable, caps.Pure derives CanEqual {
+  abstract class Name extends Designator, Showable, Pure derives CanEqual {
 
     /** A type for names of the same kind as this name */
     type ThisName <: Name

tests/pos-with-compiler-cc/dotc/core/Phases.scala

@@ -285,7 +285,7 @@ object Phases {
     final def isTyper(phase: Phase): Boolean = phase.id == typerPhase.id
   }
 
-  abstract class Phase extends caps.Pure {
+  abstract class Phase extends Pure {
 
     /** A name given to the `Phase` that can be used to debug the compiler. For
      *  instance, it is possible to print trees after a given phase using:

tests/pos-with-compiler-cc/dotc/core/Scopes.scala

@@ -60,7 +60,7 @@ object Scopes {
    *  or to delete them. These methods are provided by subclass
    *  MutableScope.
    */
-  abstract class Scope extends printing.Showable, caps.Pure {
+  abstract class Scope extends printing.Showable, Pure {
 
     /** The last scope-entry from which all others are reachable via `prev` */
     private[dotc] def lastEntry: ScopeEntry | Null

tests/pos-with-compiler-cc/dotc/core/TypeErrors.scala

@@ -15,7 +15,7 @@ import ast.untpd
 import config.Printers.cyclicErrors
 import language.experimental.pureFunctions
 
-abstract class TypeError(using creationContext: DetachedContext) extends Exception(""), caps.Pure:
+abstract class TypeError(using creationContext: DetachedContext) extends Exception(""), Pure:
 
   /** Convert to message. This takes an additional Context, so that we
    *  use the context when the message is first produced, i.e. when the TypeError

tests/pos-with-compiler-cc/dotc/core/Types.scala

@@ -91,7 +91,7 @@ object Types {
    *
    *  Note: please keep in sync with copy in `docs/docs/internals/type-system.md`.
    */
-  abstract class Type extends Hashable, printing.Showable, caps.Pure {
+  abstract class Type extends Hashable, printing.Showable, Pure {
 
 // ----- Tests -----------------------------------------------------
 

tests/pos-with-compiler-cc/dotc/core/classfile/ClassfileParser.scala

@@ -28,7 +28,7 @@ import annotation.retains
 
 object ClassfileParser {
   /** Marker trait for unpicklers that can be embedded in classfiles. */
-  trait Embedded extends caps.Pure
+  trait Embedded extends Pure
 
   /** Indicate that there is nothing to unpickle and the corresponding symbols can
     * be invalidated. */

tests/pos-with-compiler-cc/dotc/interactive/Completion.scala

@@ -538,7 +538,7 @@ object Completion {
   /** Temporary data structure representing denotations with the same name introduced in a given scope
    *  as a member of a type, by a local definition or by an import clause
    */
-  private case class ScopedDenotations(denots: Seq[SingleDenotation], ctx: DetachedContext) extends caps.Pure
+  private case class ScopedDenotations(denots: Seq[SingleDenotation], ctx: DetachedContext) extends Pure
 
   /**
    * The completion mode: defines what kinds of symbols should be included in the completion

tests/pos-with-compiler-cc/dotc/parsing/Scanners.scala

@@ -1554,7 +1554,7 @@ object Scanners {
    *   InBraces    a pair of braces { ... }
    *   Indented    a pair of <indent> ... <outdent> tokens
    */
-  abstract class Region(val closedBy: Token) extends caps.Pure:
+  abstract class Region(val closedBy: Token) extends Pure:
 
    /** The region enclosing this one, or `null` for the outermost region */
     def outer: Region | Null

tests/pos-with-compiler-cc/dotc/printing/Highlighting.scala

@@ -28,7 +28,7 @@ object Highlighting {
       else mod + super.show
   }
 
-  case class HighlightBuffer(hl: Highlight)(using DetachedContext) extends caps.Pure {
+  case class HighlightBuffer(hl: Highlight)(using DetachedContext) extends Pure {
     private val buffer = new mutable.ListBuffer[String]
 
     buffer += hl.show

tests/pos-with-compiler-cc/dotc/printing/Printer.scala

@@ -15,7 +15,7 @@ import scala.annotation.internal.sharable
 
 /** The base class of all printers
  */
-abstract class Printer extends caps.Pure {
+abstract class Printer extends Pure {
 
   private var prec: Precedence = GlobalPrec
 

tests/pos-with-compiler-cc/dotc/profile/AsyncHelper.scala

@@ -9,7 +9,7 @@ import java.util.concurrent.atomic.AtomicInteger
 import dotty.tools.dotc.core.Phases.Phase
 import dotty.tools.dotc.core.Contexts._
 
-sealed trait AsyncHelper extends caps.Pure {
+sealed trait AsyncHelper extends Pure {
 
   def newUnboundedQueueFixedThreadPool
   (nThreads: Int,

tests/pos-with-compiler-cc/dotc/profile/Profiler.scala

@@ -65,7 +65,7 @@ case class ProfileRange(start: ProfileSnap, end:ProfileSnap, phase:Phase, purpos
   def retainedHeapMB: Double = toMegaBytes(end.heapBytes - start.heapBytes)
 }
 
-sealed trait Profiler extends caps.Pure {
+sealed trait Profiler extends Pure {
 
   def finished(): Unit
 

tests/pos-with-compiler-cc/dotc/transform/CapturedVars.scala

@@ -34,7 +34,7 @@ class CapturedVars extends MiniPhase with IdentityDenotTransformer:
   override def initContext(ctx: FreshContext): Unit =
     Captured = ctx.addLocation(util.ReadOnlySet.empty)
 
-  private class RefInfo(using DetachedContext) extends caps.Pure {
+  private class RefInfo(using DetachedContext) extends Pure {
     /** The classes for which a Ref type exists. */
     val refClassKeys: collection.Set[Symbol] =
       defn.ScalaNumericValueClasses() `union` Set(defn.BooleanClass, defn.ObjectClass)