Create dotty staging library

Author: nicolasstucki

.drone.yml

@@ -40,7 +40,7 @@ steps:
   depends_on: [ clone ]
   commands:
   - cp -R . /tmp/2/ && cd /tmp/2/
-  - ./project/scripts/sbt ";dotty-bootstrapped/compile ;dotty-bootstrapped/test ;sjsSandbox/run;sjsSandbox/test;sjsJUnitTests/test"
+  - ./project/scripts/sbt ";dotty-bootstrapped/compile ;dotty-bootstrapped/test ;dotty-staging/test ;sjsSandbox/run;sjsSandbox/test;sjsJUnitTests/test"
   - ./project/scripts/bootstrapCmdTests
 
 - name: community_build

build.sbt

@@ -10,6 +10,7 @@ val `dotty-library-bootstrapped` = Build.`dotty-library-bootstrapped`
 val `dotty-library-bootstrappedJS` = Build.`dotty-library-bootstrappedJS`
 val `dotty-sbt-bridge` = Build.`dotty-sbt-bridge`
 val `dotty-sbt-bridge-tests` = Build.`dotty-sbt-bridge-tests`
+val `dotty-staging` = Build.`dotty-staging`
 val `dotty-language-server` = Build.`dotty-language-server`
 val `dotty-bench` = Build.`dotty-bench`
 val `dotty-bench-bootstrapped` = Build.`dotty-bench-bootstrapped`

compiler/src/dotty/tools/dotc/consumetasty/ConsumeTasty.scala

@@ -1,12 +1,14 @@
 package dotty.tools.dotc.consumetasty
 
+import java.net.URLClassLoader
+
 import dotty.tools.dotc
 import dotty.tools.dotc.core.Contexts._
-import dotty.tools.dotc.quoted.QuoteDriver
 
 import scala.tasty.file.TastyConsumer
 
 object ConsumeTasty {
+
   def apply(classpath: String, classes: List[String], tastyConsumer: TastyConsumer): Unit = {
     if (classes.isEmpty)
       throw new IllegalArgumentException("Parameter classes should no be empty")
@@ -16,9 +18,23 @@ object ConsumeTasty {
         new TastyFromClass(tastyConsumer)
     }
 
-    val currentClasspath = QuoteDriver.currentClasspath(getClass.getClassLoader)
+    val currentClasspath = getCurrentClasspath(getClass.getClassLoader)
     import java.io.File.{ pathSeparator => sep }
     val args = "-from-tasty" :: "-Yretain-trees" :: "-classpath" :: s"$classpath$sep$currentClasspath" :: classes
     (new Consume).process(args.toArray)
   }
+
+  private def getCurrentClasspath(cl: ClassLoader): String = {
+    val classpath0 = System.getProperty("java.class.path")
+    cl match {
+      case cl: URLClassLoader =>
+        // Loads the classes loaded by this class loader
+        // When executing `run` or `test` in sbt the classpath is not in the property java.class.path
+        import java.nio.file.Paths
+        val newClasspath = cl.getURLs.map(url => Paths.get(url.toURI).toString)
+        newClasspath.mkString("", java.io.File.pathSeparator, if (classpath0 == "") "" else java.io.File.pathSeparator + classpath0)
+      case _ => classpath0
+    }
+  }
+
 }

compiler/src/dotty/tools/dotc/core/quoted/PickledQuotes.scala

@@ -14,7 +14,7 @@ import dotty.tools.dotc.core.Types._
 import dotty.tools.dotc.core.tasty.TreePickler.Hole
 import dotty.tools.dotc.core.tasty.{PositionPickler, TastyPickler, TastyPrinter, TastyString}
 import dotty.tools.dotc.core.tasty.TreeUnpickler.UnpickleMode
-import dotty.tools.dotc.quoted.ToolboxImpl
+import dotty.tools.dotc.quoted.QuoteContext
 import dotty.tools.dotc.tastyreflect.ReflectionImpl
 
 import scala.internal.quoted._
@@ -38,14 +38,14 @@ object PickledQuotes {
   /** Transform the expression into its fully spliced Tree */
   def quotedExprToTree[T](expr: quoted.Expr[T])(implicit ctx: Context): Tree = {
     val expr1 = expr.asInstanceOf[TastyTreeExpr[Tree]]
-    ToolboxImpl.checkScopeId(expr1.scopeId)
+    QuoteContext.checkScopeId(expr1.scopeId)
     healOwner(expr1.tree)
   }
 
   /** Transform the expression into its fully spliced TypeTree */
   def quotedTypeToTree(tpe: quoted.Type[_])(implicit ctx: Context): Tree = {
     val tpe1 = tpe.asInstanceOf[TreeType[Tree]]
-    ToolboxImpl.checkScopeId(tpe1.scopeId)
+    QuoteContext.checkScopeId(tpe1.scopeId)
     healOwner(tpe1.typeTree)
   }
 

compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala

@@ -31,7 +31,7 @@ import scala.collection.mutable.ListBuffer
 import scala.collection.mutable
 import config.Printers.pickling
 import core.quoted.PickledQuotes
-import dotty.tools.dotc.quoted.ToolboxImpl
+import dotty.tools.dotc.quoted.QuoteContext
 
 import scala.quoted
 import scala.internal.quoted.{TastyTreeExpr, TreeType}
@@ -1274,8 +1274,8 @@ class TreeUnpickler(reader: TastyReader,
       val args = until(end)(readTerm())
       val splice = splices(idx)
       def wrap(arg: Tree) =
-        if (arg.isTerm) given (qctx: scala.quoted.QuoteContext) => new TastyTreeExpr(arg, ToolboxImpl.scopeId)
-        else new TreeType(arg, ToolboxImpl.scopeId)
+        if (arg.isTerm) given (qctx: scala.quoted.QuoteContext) => new TastyTreeExpr(arg, QuoteContext.scopeId)
+        else new TreeType(arg, QuoteContext.scopeId)
       val reifiedArgs = args.map(wrap)
       val filled = if (isType) {
         val quotedType = splice.asInstanceOf[Seq[Any] => quoted.Type[_]](reifiedArgs)

compiler/src/dotty/tools/dotc/quoted/ExprCompilationUnit.scala

@@ -8,4 +8,16 @@ object QuoteContext {
   def apply() given Context: scala.quoted.QuoteContext =
     new scala.quoted.QuoteContext(ReflectionImpl(the[Context]))
 
+  type ScopeId = Int
+
+  private[dotty] def checkScopeId(id: ScopeId) given Context: Unit = {
+    if (id != scopeId)
+      throw new scala.quoted.ScopeException("Cannot call `scala.quoted.staging.run(...)` within a macro or another `run(...)`")
+  }
+
+  // TODO Explore more fine grained scope ids.
+  //      This id can only differentiate scope extrusion from one compiler instance to another.
+  private[dotty] def scopeId given Context: ScopeId =
+    the[Context].outersIterator.toList.last.hashCode()
+
 }

compiler/src/dotty/tools/dotc/quoted/QuoteContext.scala

@@ -25,7 +25,7 @@ import dotty.tools.repl.AbstractFileClassLoader
 
 import scala.reflect.ClassTag
 
-import dotty.tools.dotc.quoted.{QuoteContext, ToolboxImpl}
+import dotty.tools.dotc.quoted.QuoteContext
 
 /** Utility class to splice quoted expressions */
 object Splicer {
@@ -251,10 +251,10 @@ object Splicer {
     }
 
     private def interpretQuote(tree: Tree)(implicit env: Env): Object =
-      new scala.internal.quoted.TastyTreeExpr(Inlined(EmptyTree, Nil, tree).withSpan(tree.span), ToolboxImpl.scopeId)
+      new scala.internal.quoted.TastyTreeExpr(Inlined(EmptyTree, Nil, tree).withSpan(tree.span), QuoteContext.scopeId)
 
     private def interpretTypeQuote(tree: Tree)(implicit env: Env): Object =
-      new scala.internal.quoted.TreeType(tree, ToolboxImpl.scopeId)
+      new scala.internal.quoted.TreeType(tree, QuoteContext.scopeId)
 
     private def interpretLiteral(value: Any)(implicit env: Env): Object =
       value.asInstanceOf[Object]

compiler/src/dotty/tools/dotc/quoted/ToolboxImpl.scala

@@ -50,6 +50,9 @@ object Properties {
   /** dotty-compiler jar */
   def dottyCompiler: String = sys.props("dotty.tests.classes.dottyCompiler")
 
+  /** dotty-staging jar */
+  def dottyStaging: String = sys.props("dotty.tests.classes.dottyStaging")
+
   /** compiler-interface jar */
   def compilerInterface: String = sys.props("dotty.tests.classes.compilerInterface")
 

compiler/src/dotty/tools/dotc/transform/Splicer.scala

@@ -39,6 +39,7 @@ class BootstrappedOnlyCompilationTests extends ParallelTesting {
     implicit val testGroup: TestGroup = TestGroup("compilePosWithCompiler")
     aggregateTests(
       compileFilesInDir("tests/pos-with-compiler", withCompilerOptions),
+      compileFilesInDir("tests/pos-staging", withStagingOptions),
       compileDir("compiler/src/dotty/tools/dotc/ast", withCompilerOptions),
       compileDir("compiler/src/dotty/tools/dotc/config", withCompilerOptions),
       compileDir("compiler/src/dotty/tools/dotc/core", withCompilerOptions),
@@ -103,7 +104,10 @@ class BootstrappedOnlyCompilationTests extends ParallelTesting {
 
   @Test def negWithCompiler: Unit = {
     implicit val testGroup: TestGroup = TestGroup("compileNegWithCompiler")
-    compileFilesInDir("tests/neg-with-compiler", withCompilerOptions).checkExpectedErrors()
+    aggregateTests(
+      compileFilesInDir("tests/neg-with-compiler", withCompilerOptions),
+      compileFilesInDir("tests/neg-staging", withStagingOptions),
+    ).checkExpectedErrors()
   }
 
   // Run tests -----------------------------------------------------------------
@@ -120,6 +124,7 @@ class BootstrappedOnlyCompilationTests extends ParallelTesting {
     implicit val testGroup: TestGroup = TestGroup("runWithCompiler")
     aggregateTests(
       compileFilesInDir("tests/run-with-compiler", withCompilerOptions),
+      compileFilesInDir("tests/run-staging", withStagingOptions),
       compileDir("tests/run-with-compiler-custom-args/tasty-interpreter", withCompilerOptions)
     ).checkRuns()
   }

compiler/test/dotty/Properties.scala

@@ -9,12 +9,12 @@ import dotty.tools.dotc.reporting.MessageRendering
 import org.junit.{After, Before}
 
 
-class ReplTest private (out: ByteArrayOutputStream) extends ReplDriver(
-  Array("-classpath", TestConfiguration.basicClasspath, "-color:never"),
-  new PrintStream(out)
-) with MessageRendering {
+class ReplTest private (settings: Array[String], out: ByteArrayOutputStream)
+    extends ReplDriver(settings, new PrintStream(out)) with MessageRendering {
 
-  def this() = this(new ByteArrayOutputStream)
+  def this(classpath: String) = this(Array("-classpath", classpath, "-color:never"), new ByteArrayOutputStream)
+
+  def this() = this(TestConfiguration.basicClasspath)
 
   /** Get the stored output from `out`, resetting the buffer */
   def storedOutput(): String = {

compiler/test/dotty/tools/dotc/BootstrappedOnlyCompilationTests.scala

@@ -11,19 +11,19 @@ import org.junit.experimental.categories.Category
 
 import scala.collection.mutable.ArrayBuffer
 import scala.io.Source
-
 import dotc.reporting.MessageRendering
+import dotty.tools.vulpix.TestConfiguration
 
 /** Runs all tests contained in `compiler/test-resources/repl/` */
-class ScriptedTests extends ReplTest with MessageRendering {
+class ScriptedReplTests(classpath: String) extends ReplTest(classpath) with MessageRendering {
 
-  private def scripts(path: String): Array[JFile] = {
+  protected def scripts(path: String): Array[JFile] = {
     val dir = new JFile(getClass.getResource(path).getPath)
     assert(dir.exists && dir.isDirectory, "Couldn't load scripts dir")
     dir.listFiles
   }
 
-  private def testFile(f: JFile): Unit = {
+  protected def testFile(f: JFile): Unit = {
     val prompt = "scala>"
     val lines = Source.fromFile(f, "UTF-8").getLines().buffered
 
@@ -89,11 +89,13 @@ class ScriptedTests extends ReplTest with MessageRendering {
       fail(s"Error in file $f, expected output did not match actual")
     }
   }
+}
+
+/** Runs all tests contained in `compiler/test-resources/repl/` */
+class ScriptedTests extends ScriptedReplTests(TestConfiguration.basicClasspath) {
 
   @Test def replTests = scripts("/repl").foreach(testFile)
 
   @Test def typePrinterTests = scripts("/type-printer").foreach(testFile)
 
-  @Category(Array(classOf[BootstrappedOnlyTests]))
-  @Test def replMacrosTests = scripts("/repl-macros").foreach(testFile)
 }

compiler/test/dotty/tools/repl/ReplTest.scala

@@ -35,6 +35,18 @@ object TestConfiguration {
     Properties.dottyCompiler
   ))
 
+  val withStagingClasspath = mkClasspath(List(
+    Properties.scalaLibrary,
+    Properties.scalaAsm,
+    Properties.jlineTerminal,
+    Properties.jlineReader,
+    Properties.compilerInterface,
+    Properties.dottyInterfaces,
+    Properties.dottyLibrary,
+    Properties.dottyStaging,
+    Properties.dottyCompiler
+  ))
+
   def mkClasspath(classpaths: List[String]): String =
     classpaths.map({ p =>
       val file = new java.io.File(p)
@@ -48,6 +60,8 @@ object TestConfiguration {
   val defaultOptions = TestFlags(basicClasspath, commonOptions)
   val withCompilerOptions =
     defaultOptions.withClasspath(withCompilerClasspath).withRunClasspath(withCompilerClasspath)
+  val withStagingOptions =
+    defaultOptions.withClasspath(withStagingClasspath).withRunClasspath(withStagingClasspath)
   val allowDeepSubtypes = defaultOptions without "-Yno-deep-subtypes"
   val allowDoubleBindings = defaultOptions without "-Yno-double-bindings"
   val picklingOptions = defaultOptions and (

compiler/test/dotty/tools/repl/ScriptedTests.scala

@@ -107,6 +107,7 @@ find_lib () {
 DOTTY_COMP=$(find_lib "*dotty-compiler*")
 DOTTY_INTF=$(find_lib "*dotty-interfaces*")
 DOTTY_LIB=$(find_lib "*dotty-library*")
+DOTTY_STAGING=$(find_lib "*dotty-staging*")
 SCALA_ASM=$(find_lib "*scala-asm*")
 SCALA_LIB=$(find_lib "*scala-library*")
 SBT_INTF=$(find_lib "*compiler-interface*")

compiler/test/dotty/tools/vulpix/TestConfiguration.scala

@@ -102,7 +102,7 @@ elif [ $execute_repl == true ] || [ ${#residual_args[@]} -ne 0 ]; then
     echo "warning: multiple classpaths are found, dotr only use the last one."
   fi
   if [ $with_compiler == true ]; then
-    cp_arg+="$PSEP$DOTTY_COMP$PSEP$DOTTY_INTF$PSEP$SCALA_ASM"
+    cp_arg+="$PSEP$DOTTY_COMP$PSEP$DOTTY_INTF$PSEP$SCALA_ASM$DOTTY_STAGING"
   fi
   eval exec "\"$JAVACMD\"" "$DEBUG"  "-classpath \"$cp_arg\"" "${jvm_options[@]}" "${residual_args[@]}"
 else