Add Stable Presentation Compiler (#17528)

This PR adds a stable presentation compiler implementation to dotty.
This will ensure that each future version of Scala 3 is shipped with
presentation compiler compiled against it, guaranteeing that IDE will
support it. It will also improve support for projects relying on
nonbootstrapped compiler such as scaladoc or dotty-language-server, as
it is now possible to easily publish presentation compiler for those
versions from dotty repository.

It also adds vast of tests suits ported from metals, which will also
help to detect unintended changes before they are merged. More
information about this initiative can be found here:
https://contributors.scala-lang.org/t/stable-presentation-compiler-api/6139

Author: rochala

NOTICE.md

@@ -89,15 +89,19 @@ major authors were omitted by oversight.
     details.
 
   * dotty.tools.dotc.coverage: Coverage instrumentation utilities have been
-    adapted from the scoverage plugin for scala 2 [5], which is under the
+    adapted from the scoverage plugin for scala 2 [4], which is under the
     Apache 2.0 license.
 
+  * dooty.tools.pc: Presentation compiler implementation adapted from
+    scalameta/metals [5] mtags module, which is under the Apache 2.0 license.
+
   * The Dotty codebase contains parts which are derived from
-    the ScalaPB protobuf library [4], which is under the Apache 2.0 license.
+    the ScalaPB protobuf library [6], which is under the Apache 2.0 license.
 
 
 [1] https://github.com/scala/scala
 [2] https://github.com/adriaanm/scala/tree/sbt-api-consolidate/src/compiler/scala/tools/sbt
 [3] https://github.com/sbt/sbt/tree/0.13/compile/interface/src/main/scala/xsbt
-[4] https://github.com/lampepfl/dotty/pull/5783/files
-[5] https://github.com/scoverage/scalac-scoverage-plugin
+[4] https://github.com/scoverage/scalac-scoverage-plugin
+[5] https://github.com/scalameta/metals
+[6] https://github.com/lampepfl/dotty/pull/5783/files

build.sbt

@@ -29,6 +29,8 @@ val `scala3-bench-run` = Build.`scala3-bench-run`
 val dist = Build.dist
 val `community-build` = Build.`community-build`
 val `sbt-community-build` = Build.`sbt-community-build`
+val `scala3-presentation-compiler` = Build.`scala3-presentation-compiler`
+val `scala3-presentation-compiler-bootstrapped` = Build.`scala3-presentation-compiler-bootstrapped`
 
 val sjsSandbox = Build.sjsSandbox
 val sjsJUnitTests = Build.sjsJUnitTests

compiler/src/dotty/tools/dotc/CompilationUnit.scala

@@ -5,6 +5,7 @@ import core._
 import Contexts._
 import SymDenotations.ClassDenotation
 import Symbols._
+import Comments.Comment
 import util.{FreshNameCreator, SourceFile, NoSource}
 import util.Spans.Span
 import ast.{tpd, untpd}
@@ -69,6 +70,9 @@ class CompilationUnit protected (val source: SourceFile) {
   /** Can this compilation unit be suspended */
   def isSuspendable: Boolean = true
 
+  /** List of all comments present in this compilation unit */
+  var comments: List[Comment] = Nil
+
   /** Suspends the compilation unit by thowing a SuspendException
    *  and recording the suspended compilation unit
    */

compiler/src/dotty/tools/dotc/interactive/InteractiveDriver.scala

@@ -145,7 +145,7 @@ class InteractiveDriver(val settings: List[String]) extends Driver {
     (fromSource ++ fromClassPath).distinct
   }
 
-  def run(uri: URI, sourceCode: String): List[Diagnostic] = run(uri, toSource(uri, sourceCode))
+  def run(uri: URI, sourceCode: String): List[Diagnostic] = run(uri, SourceFile.virtual(uri, sourceCode))
 
   def run(uri: URI, source: SourceFile): List[Diagnostic] = {
     import typer.ImportInfo._
@@ -297,9 +297,6 @@ class InteractiveDriver(val settings: List[String]) extends Driver {
     cleanupTree(tree)
   }
 
-  private def toSource(uri: URI, sourceCode: String): SourceFile =
-    SourceFile.virtual(Paths.get(uri).toString, sourceCode)
-
   /**
    * Initialize this driver and compiler.
    *
@@ -323,7 +320,7 @@ object InteractiveDriver {
     else
       try
         // We don't use file.file here since it'll be null
-        // for the VirtualFiles created by InteractiveDriver#toSource
+        // for the VirtualFiles created by SourceFile#virtual
         // TODO: To avoid these round trip conversions, we could add an
         // AbstractFile#toUri method and implement it by returning a constant
         // passed as a parameter to a constructor of VirtualFile

compiler/src/dotty/tools/dotc/parsing/ParserPhase.scala

@@ -30,6 +30,7 @@ class Parser extends Phase {
         val p = new Parsers.Parser(unit.source)
         //  p.in.debugTokenStream = true
         val tree = p.parse()
+        ctx.compilationUnit.comments = p.in.comments
         if (p.firstXmlPos.exists && !firstXmlPos.exists)
           firstXmlPos = p.firstXmlPos
         tree

compiler/src/dotty/tools/dotc/parsing/Scanners.scala

@@ -227,11 +227,11 @@ object Scanners {
       */
     private var docstringMap: SortedMap[Int, Comment] = SortedMap.empty
 
-    /* A Buffer for comment positions */
-    private val commentPosBuf = new mutable.ListBuffer[Span]
+    /* A Buffer for comments */
+    private val commentBuf = new mutable.ListBuffer[Comment]
 
-    /** Return a list of all the comment positions */
-    def commentSpans: List[Span] = commentPosBuf.toList
+    /** Return a list of all the comments */
+    def comments: List[Comment] = commentBuf.toList
 
     private def addComment(comment: Comment): Unit = {
       val lookahead = lookaheadReader()
@@ -246,7 +246,7 @@ object Scanners {
     def getDocComment(pos: Int): Option[Comment] = docstringMap.get(pos)
 
     /** A buffer for comments */
-    private val commentBuf = CharBuffer(initialCharBufferSize)
+    private val currentCommentBuf = CharBuffer(initialCharBufferSize)
 
     def toToken(identifier: SimpleName): Token =
       def handleMigration(keyword: Token): Token =
@@ -523,7 +523,7 @@ object Scanners {
      *
      *      The following tokens can start an indentation region:
      *
-     *         :  =  =>  <-  if  then  else  while  do  try  catch  
+     *         :  =  =>  <-  if  then  else  while  do  try  catch
      *         finally  for  yield  match  throw  return  with
      *
      *      Inserting an INDENT starts a new indentation region with the indentation of the current
@@ -1019,7 +1019,7 @@ object Scanners {
 
     private def skipComment(): Boolean = {
       def appendToComment(ch: Char) =
-        if (keepComments) commentBuf.append(ch)
+        if (keepComments) currentCommentBuf.append(ch)
       def nextChar() = {
         appendToComment(ch)
         Scanner.this.nextChar()
@@ -1047,9 +1047,9 @@ object Scanners {
       def finishComment(): Boolean = {
         if (keepComments) {
           val pos = Span(start, charOffset - 1, start)
-          val comment = Comment(pos, commentBuf.toString)
-          commentBuf.clear()
-          commentPosBuf += pos
+          val comment = Comment(pos, currentCommentBuf.toString)
+          currentCommentBuf.clear()
+          commentBuf += comment
 
           if (comment.isDocComment)
             addComment(comment)
@@ -1065,7 +1065,7 @@ object Scanners {
       else if (ch == '*') { nextChar(); skipComment(); finishComment() }
       else {
         // This was not a comment, remove the `/` from the buffer
-        commentBuf.clear()
+        currentCommentBuf.clear()
         false
       }
     }

compiler/src/dotty/tools/dotc/printing/PlainPrinter.scala

@@ -179,7 +179,7 @@ class PlainPrinter(_ctx: Context) extends Printer {
         if (printWithoutPrefix.contains(tp.symbol))
           toText(tp.name)
         else
-          toTextPrefix(tp.prefix) ~ selectionString(tp)
+          toTextPrefixOf(tp) ~ selectionString(tp)
       case tp: TermParamRef =>
         ParamRefNameString(tp) ~ lambdaHash(tp.binder) ~ ".type"
       case tp: TypeParamRef =>
@@ -353,7 +353,7 @@ class PlainPrinter(_ctx: Context) extends Printer {
   def toTextRef(tp: SingletonType): Text = controlled {
     tp match {
       case tp: TermRef =>
-        toTextPrefix(tp.prefix) ~ selectionString(tp)
+        toTextPrefixOf(tp) ~ selectionString(tp)
       case tp: ThisType =>
         nameString(tp.cls) + ".this"
       case SuperType(thistpe: SingletonType, _) =>
@@ -375,15 +375,6 @@ class PlainPrinter(_ctx: Context) extends Printer {
     }
   }
 
-  /** The string representation of this type used as a prefix, including separator */
-  def toTextPrefix(tp: Type): Text = controlled {
-    homogenize(tp) match {
-      case NoPrefix => ""
-      case tp: SingletonType => toTextRef(tp) ~ "."
-      case tp => trimPrefix(toTextLocal(tp)) ~ "#"
-    }
-  }
-
   def toTextCaptureRef(tp: Type): Text =
     homogenize(tp) match
       case tp: TermRef if tp.symbol == defn.captureRoot => Str("cap")
@@ -393,6 +384,15 @@ class PlainPrinter(_ctx: Context) extends Printer {
   protected def isOmittablePrefix(sym: Symbol): Boolean =
     defn.unqualifiedOwnerTypes.exists(_.symbol == sym) || isEmptyPrefix(sym)
 
+  /** The string representation of type prefix, including separator */
+  def toTextPrefixOf(tp: NamedType): Text = controlled {
+      homogenize(tp.prefix) match {
+        case NoPrefix => ""
+        case tp: SingletonType => toTextRef(tp) ~ "."
+        case tp => trimPrefix(toTextLocal(tp)) ~ "#"
+      }
+  }
+
   protected def isEmptyPrefix(sym: Symbol): Boolean =
     sym.isEffectiveRoot || sym.isAnonymousClass || sym.name.isReplWrapperName
 

compiler/src/dotty/tools/dotc/printing/Printer.scala

@@ -4,7 +4,7 @@ package printing
 
 import core._
 import Texts._, ast.Trees._
-import Types.{Type, SingletonType, LambdaParam},
+import Types.{Type, SingletonType, LambdaParam, NamedType},
        Symbols.Symbol, Scopes.Scope, Constants.Constant,
        Names.Name, Denotations._, Annotations.Annotation, Contexts.Context
 import typer.Implicits.SearchResult
@@ -101,7 +101,7 @@ abstract class Printer {
   def toTextRef(tp: SingletonType): Text
 
   /** Textual representation of a prefix of some reference, ending in `.` or `#` */
-  def toTextPrefix(tp: Type): Text
+  def toTextPrefixOf(tp: NamedType): Text
 
   /** Textual representation of a reference in a capture set */
   def toTextCaptureRef(tp: Type): Text

compiler/src/dotty/tools/dotc/printing/RefinedPrinter.scala

@@ -117,21 +117,22 @@ class RefinedPrinter(_ctx: Context) extends PlainPrinter(_ctx) {
     }
   }
 
-  override def toTextPrefix(tp: Type): Text = controlled {
+  override def toTextPrefixOf(tp: NamedType): Text = controlled {
     def isOmittable(sym: Symbol) =
       if printDebug then false
       else if homogenizedView then isEmptyPrefix(sym) // drop <root> and anonymous classes, but not scala, Predef.
       else if sym.isPackageObject then isOmittablePrefix(sym.owner)
       else isOmittablePrefix(sym)
-    tp match {
-      case tp: ThisType if isOmittable(tp.cls) =>
+
+    tp.prefix match {
+      case thisType: ThisType if isOmittable(thisType.cls) =>
         ""
-      case tp @ TermRef(pre, _) =>
-        val sym = tp.symbol
-        if sym.isPackageObject && !homogenizedView && !printDebug then toTextPrefix(pre)
+      case termRef @ TermRef(pre, _) =>
+        val sym = termRef.symbol
+        if sym.isPackageObject && !homogenizedView && !printDebug then toTextPrefixOf(termRef)
         else if (isOmittable(sym)) ""
-        else super.toTextPrefix(tp)
-      case _ => super.toTextPrefix(tp)
+        else super.toTextPrefixOf(tp)
+      case _ => super.toTextPrefixOf(tp)
     }
   }
 
@@ -427,8 +428,7 @@ class RefinedPrinter(_ctx: Context) extends PlainPrinter(_ctx) {
       case id @ Ident(name) =>
         val txt = tree.typeOpt match {
           case tp: NamedType if name != nme.WILDCARD =>
-            val pre = if (tp.symbol.is(JavaStatic)) tp.prefix.widen else tp.prefix
-            toTextPrefix(pre) ~ withPos(selectionString(tp), tree.sourcePos)
+            toTextPrefixOf(tp) ~ withPos(selectionString(tp), tree.sourcePos)
           case _ =>
             toText(name)
         }

compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala

@@ -83,8 +83,8 @@ object SyntaxHighlighting {
         }
       }
 
-      for (span <- scanner.commentSpans)
-        highlightPosition(span, CommentColor)
+      for (comment <- scanner.comments)
+        highlightPosition(comment.span, CommentColor)
 
       object TreeHighlighter extends untpd.UntypedTreeTraverser {
         import untpd._

compiler/src/dotty/tools/dotc/typer/ImportSuggestions.scala

@@ -330,7 +330,7 @@ trait ImportSuggestions:
     def importString(ref: TermRef): String =
       val imported =
         if ref.symbol.is(ExtensionMethod) then
-          s"${ctx.printer.toTextPrefix(ref.prefix).show}${ref.symbol.name}"
+          s"${ctx.printer.toTextPrefixOf(ref).show}${ref.symbol.name}"
         else
           ctx.printer.toTextRef(ref).show
       s"  import $imported"

compiler/src/dotty/tools/dotc/util/DiffUtil.scala

@@ -70,7 +70,9 @@ object DiffUtil {
    *         differences are highlighted.
    */
   def mkColoredLineDiff(expected: Seq[String], actual: Seq[String]): String = {
-    val expectedSize = EOF.length max expected.maxBy(_.length).length
+    val longestExpected = expected.map(_.length).maxOption.getOrElse(0)
+    val longestActual = actual.map(_.length).maxOption.getOrElse(0)
+    val expectedSize = EOF.length max longestActual max longestExpected
     actual.padTo(expected.length, "").zip(expected.padTo(actual.length, "")).map { case (act, exp) =>
       mkColoredLineDiff(exp, act, expectedSize)
     }.mkString(System.lineSeparator)
@@ -101,11 +103,75 @@ object DiffUtil {
         case Deleted(str) => deleted(str)
       }.mkString
 
+    (expectedDiff, actualDiff)
     val pad = " " * 0.max(expectedSize - expected.length)
 
     expectedDiff + pad + "  |  " + actualDiff
   }
 
+  private def ensureLineSeparator(str: String): String =
+    if str.endsWith(System.lineSeparator) then
+      str
+    else
+      str + System.lineSeparator
+
+  /**
+   * Returns a colored diffs by comparison of lines instead of tokens.
+   * It will automatically group subsequential pairs of `Insert` and `Delete`
+   * in order to improve the readability
+   *
+   * @param expected The expected lines
+   * @param actual   The actual lines
+   * @return A string with colored diffs between `expected` and `actual` grouped whenever possible
+   */
+  def mkColoredHorizontalLineDiff(expected: String, actual: String): String = {
+    val indent = 2
+    val tab = " " * indent
+    val insertIndent = "+" ++ (" " * (indent - 1))
+    val deleteIndent = "-" ++ (" " * (indent - 1))
+
+    if actual.isEmpty then
+      (expected.linesIterator.map(line => added(insertIndent + line)).toList :+ deleted("--- EMPTY OUTPUT ---"))
+        .map(ensureLineSeparator).mkString
+    else if expected.isEmpty then
+      (added("--- NO VALUE EXPECTED ---") +: actual.linesIterator.map(line => deleted(deleteIndent + line)).toList)
+        .map(ensureLineSeparator).mkString
+    else
+      lazy val diff = {
+        val expectedTokens = expected.linesWithSeparators.toArray
+        val actualTokens = actual.linesWithSeparators.toArray
+        hirschberg(actualTokens, expectedTokens)
+      }.toList
+
+      val transformedDiff = diff.flatMap {
+        case Modified(original, str) => Seq(
+          Inserted(ensureLineSeparator(original)), Deleted(ensureLineSeparator(str))
+        )
+        case other => Seq(other)
+      }
+
+      val zipped = transformedDiff zip transformedDiff.drop(1)
+
+      val (acc, inserts, deletions) = zipped.foldLeft((Seq[Patch](), Seq[Inserted](), Seq[Deleted]())): (acc, patches) =>
+        val (currAcc, inserts, deletions) = acc
+        patches match
+          case (currentPatch: Inserted, nextPatch: Deleted) =>
+            (currAcc, inserts :+ currentPatch, deletions)
+          case (currentPatch: Deleted, nextPatch: Inserted) =>
+            (currAcc, inserts, deletions :+ currentPatch)
+          case (currentPatch, nextPatch) =>
+            (currAcc :++ inserts :++ deletions :+ currentPatch, Seq.empty, Seq.empty)
+
+      val stackedDiff = acc :++ inserts :++ deletions :+ diff.last
+
+      stackedDiff.collect {
+        case Unmodified(str) => tab + str
+        case Inserted(str) => added(insertIndent + str)
+        case Deleted(str) => deleted(deleteIndent + str)
+      }.map(ensureLineSeparator).mkString
+
+  }
+
   def mkColoredCodeDiff(code: String, lastCode: String, printDiffDel: Boolean): String = {
     val tokens = splitTokens(code, Nil).toArray
     val lastTokens = splitTokens(lastCode, Nil).toArray

compiler/src/dotty/tools/dotc/util/SourceFile.scala

@@ -16,8 +16,9 @@ import scala.collection.mutable.ArrayBuffer
 import scala.util.chaining.given
 
 import java.io.File.separator
+import java.net.URI
 import java.nio.charset.StandardCharsets
-import java.nio.file.{FileSystemException, NoSuchFileException}
+import java.nio.file.{FileSystemException, NoSuchFileException, Paths}
 import java.util.Optional
 import java.util.concurrent.atomic.AtomicInteger
 import java.util.regex.Pattern
@@ -222,6 +223,13 @@ object SourceFile {
     SourceFile(new VirtualFile(name.replace(separator, "/"), content.getBytes(StandardCharsets.UTF_8)), content.toCharArray)
       .tap(_._maybeInComplete = maybeIncomplete)
 
+  /** A helper method to create a virtual source file for given URI.
+   *  It relies on SourceFile#virtual implementation to create the virtual file.
+   */
+  def virtual(uri: URI, content: String): SourceFile =
+    val path = Paths.get(uri).toString
+    SourceFile.virtual(path, content)
+
   /** Returns the relative path of `source` within the `reference` path
    *
    *  It returns the absolute path of `source` if it is not contained in `reference`.