use new zinc api for virtualfile

Author: bishabosha

compiler/src/dotty/tools/backend/jvm/CodeGen.scala

@@ -30,7 +30,7 @@ import scala.tools.asm
 import scala.tools.asm.tree._
 import tpd._
 import dotty.tools.io.AbstractFile
-import dotty.tools.dotc.util.NoSourcePosition
+import dotty.tools.dotc.util.{NoSourcePosition, SourceFile}
 
 
 class CodeGen(val int: DottyBackendInterface, val primitives: DottyPrimitives)( val bTypes: BTypesFromSymbols[int.type]) { self =>
@@ -106,7 +106,7 @@ class CodeGen(val int: DottyBackendInterface, val primitives: DottyPrimitives)(
   }
 
   // Creates a callback that will be evaluated in PostProcessor after creating a file
-  private def onFileCreated(cls: ClassNode, claszSymbol: Symbol, sourceFile: interfaces.SourceFile): AbstractFile => Unit = clsFile => {
+  private def onFileCreated(cls: ClassNode, claszSymbol: Symbol, sourceFile: SourceFile): AbstractFile => Unit = clsFile => {
     val (fullClassName, isLocal) = atPhase(sbtExtractDependenciesPhase) {
       (ExtractDependencies.classNameAsString(claszSymbol), claszSymbol.isLocal)
     }
@@ -116,10 +116,10 @@ class CodeGen(val int: DottyBackendInterface, val primitives: DottyPrimitives)(
       ctx.compilerCallback.onClassGenerated(sourceFile, convertAbstractFile(clsFile), className)
 
     if (ctx.sbtCallback != null) {
-      val jSourceFile = sourceFile.jfile.orElse(null)
+      val jSourceFile = sourceFile.underlyingZincFile
       val cb = ctx.sbtCallback
-      if (isLocal) cb.generatedLocalClass(jSourceFile, clsFile.file)
-      else cb.generatedNonLocalClass(jSourceFile, clsFile.file, className, fullClassName)
+      if (isLocal) cb.generatedLocalClass(jSourceFile, clsFile.jpath)
+      else cb.generatedNonLocalClass(jSourceFile, clsFile.jpath, className, fullClassName)
     }
   }
 

compiler/src/dotty/tools/dotc/config/ScalaSettings.scala

@@ -325,6 +325,7 @@ private sealed trait YSettings:
   val YdebugTypeError: Setting[Boolean] = BooleanSetting("-Ydebug-type-error", "Print the stack trace when a TypeError is caught", false)
   val YdebugError: Setting[Boolean] = BooleanSetting("-Ydebug-error", "Print the stack trace when any error is caught.", false)
   val YdebugUnpickling: Setting[Boolean] = BooleanSetting("-Ydebug-unpickling", "Print the stack trace when an error occurs when reading Tasty.", false)
+  val YdebugVirtualFiles: Setting[Boolean] = BooleanSetting("-Ydebug-virtual-files", "Debug usage of virtual files, e.g. remote cache in sbt", false)
   val YtermConflict: Setting[String] = ChoiceSetting("-Yresolve-term-conflict", "strategy", "Resolve term conflicts", List("package", "object", "error"), "error")
   val Ylog: Setting[List[String]] = PhasesSetting("-Ylog", "Log operations during")
   val YlogClasspath: Setting[Boolean] = BooleanSetting("-Ylog-classpath", "Output information about what classpath is being applied.")

compiler/src/dotty/tools/dotc/core/Contexts.scala

@@ -39,22 +39,25 @@ import util.Store
 import xsbti.AnalysisCallback
 import plugins._
 import java.util.concurrent.atomic.AtomicInteger
+import java.util.Map as JMap
 import java.nio.file.InvalidPathException
 
+
 object Contexts {
 
-  private val (compilerCallbackLoc, store1) = Store.empty.newLocation[CompilerCallback]()
-  private val (sbtCallbackLoc,      store2) = store1.newLocation[AnalysisCallback]()
-  private val (printerFnLoc,        store3) = store2.newLocation[Context => Printer](new RefinedPrinter(_))
-  private val (settingsStateLoc,    store4) = store3.newLocation[SettingsState]()
-  private val (compilationUnitLoc,  store5) = store4.newLocation[CompilationUnit]()
-  private val (runLoc,              store6) = store5.newLocation[Run | Null]()
-  private val (profilerLoc,         store7) = store6.newLocation[Profiler]()
-  private val (notNullInfosLoc,     store8) = store7.newLocation[List[NotNullInfo]]()
-  private val (importInfoLoc,       store9) = store8.newLocation[ImportInfo | Null]()
-  private val (typeAssignerLoc,    store10) = store9.newLocation[TypeAssigner](TypeAssigner)
+  private val (compilerCallbackLoc,  store1) = Store.empty.newLocation[CompilerCallback]()
+  private val (sbtCallbackLoc,       store2) = store1.newLocation[AnalysisCallback]()
+  private val (printerFnLoc,         store3) = store2.newLocation[Context => Printer](new RefinedPrinter(_))
+  private val (settingsStateLoc,     store4) = store3.newLocation[SettingsState]()
+  private val (compilationUnitLoc,   store5) = store4.newLocation[CompilationUnit]()
+  private val (runLoc,               store6) = store5.newLocation[Run | Null]()
+  private val (profilerLoc,          store7) = store6.newLocation[Profiler]()
+  private val (notNullInfosLoc,      store8) = store7.newLocation[List[NotNullInfo]]()
+  private val (importInfoLoc,        store9) = store8.newLocation[ImportInfo | Null]()
+  private val (typeAssignerLoc,     store10) = store9.newLocation[TypeAssigner](TypeAssigner)
+  private val (zincVirtualFilesLoc, store11) = store10.newLocation[JMap[String, xsbti.VirtualFile] | Null]()
 
-  private val initialStore = store10
+  private val initialStore = store11
 
   /** The current context */
   inline def ctx(using ctx: Context): Context = ctx
@@ -167,6 +170,9 @@ object Contexts {
     /** The sbt callback implementation if we are run from sbt, null otherwise */
     def sbtCallback: AnalysisCallback = store(sbtCallbackLoc)
 
+    /** A map from absolute path to VirtualFile if we are run from sbt, null otherwise */
+    def zincVirtualFiles: JMap[String, xsbti.VirtualFile] | Null = store(zincVirtualFilesLoc)
+
     /** The current plain printer */
     def printerFn: Context => Printer = store(printerFnLoc)
 
@@ -665,6 +671,8 @@ object Contexts {
 
     def setCompilerCallback(callback: CompilerCallback): this.type = updateStore(compilerCallbackLoc, callback)
     def setSbtCallback(callback: AnalysisCallback): this.type = updateStore(sbtCallbackLoc, callback)
+    def setZincVirtualFiles(map: JMap[String, xsbti.VirtualFile]): this.type =
+      updateStore(zincVirtualFilesLoc, map)
     def setPrinterFn(printer: Context => Printer): this.type = updateStore(printerFnLoc, printer)
     def setSettings(settingsState: SettingsState): this.type = updateStore(settingsStateLoc, settingsState)
     def setRun(run: Run | Null): this.type = updateStore(runLoc, run)

compiler/src/dotty/tools/dotc/sbt/APIUtils.scala

@@ -37,7 +37,7 @@ object APIUtils {
   def registerDummyClass(classSym: ClassSymbol)(using Context): Unit = {
     if (ctx.sbtCallback != null) {
       val classLike = emptyClassLike(classSym)
-      ctx.sbtCallback.api(ctx.compilationUnit.source.file.file, classLike)
+      ctx.sbtCallback.api(ctx.compilationUnit.source.underlyingZincFile, classLike)
     }
   }
 

compiler/src/dotty/tools/dotc/sbt/ExtractAPI.scala

@@ -65,17 +65,17 @@ class ExtractAPI extends Phase {
 
   override def run(using Context): Unit = {
     val unit = ctx.compilationUnit
-    val sourceFile = unit.source.file
+    val sourceFile = unit.source
     if (ctx.sbtCallback != null)
-      ctx.sbtCallback.startSource(sourceFile.file)
+      ctx.sbtCallback.startSource(sourceFile.underlyingZincFile)
 
     val apiTraverser = new ExtractAPICollector
     val classes = apiTraverser.apiSource(unit.tpdTree)
     val mainClasses = apiTraverser.mainClasses
 
     if (ctx.settings.YdumpSbtInc.value) {
       // Append to existing file that should have been created by ExtractDependencies
-      val pw = new PrintWriter(File(sourceFile.jpath).changeExtension("inc").toFile
+      val pw = new PrintWriter(File(sourceFile.file.jpath).changeExtension("inc").toFile
         .bufferedWriter(append = true), true)
       try {
         classes.foreach(source => pw.println(DefaultShowAPI(source)))
@@ -85,8 +85,8 @@ class ExtractAPI extends Phase {
     if ctx.sbtCallback != null &&
       !ctx.compilationUnit.suspendedAtInliningPhase // already registered before this unit was suspended
     then
-      classes.foreach(ctx.sbtCallback.api(sourceFile.file, _))
-      mainClasses.foreach(ctx.sbtCallback.mainClass(sourceFile.file, _))
+      classes.foreach(ctx.sbtCallback.api(sourceFile.underlyingZincFile, _))
+      mainClasses.foreach(ctx.sbtCallback.mainClass(sourceFile.underlyingZincFile, _))
   }
 }
 

compiler/src/dotty/tools/dotc/sbt/ExtractDependencies.scala

@@ -4,6 +4,7 @@ package sbt
 import scala.language.unsafeNulls
 
 import java.io.File
+import java.nio.file.Path
 import java.util.{Arrays, EnumSet}
 
 import dotty.tools.dotc.ast.tpd
@@ -19,6 +20,7 @@ import dotty.tools.dotc.core.Denotations.StaleSymbol
 import dotty.tools.dotc.core.Types._
 import dotty.tools.dotc.transform.SymUtils._
 import dotty.tools.dotc.util.{SrcPos, NoSourcePosition}
+import dotty.tools.uncheckedNN
 import dotty.tools.io
 import dotty.tools.io.{AbstractFile, PlainFile, ZipArchive}
 import xsbti.UseScope
@@ -56,7 +58,7 @@ class ExtractDependencies extends Phase {
 
   override def isRunnable(using Context): Boolean = {
     def forceRun = ctx.settings.YdumpSbtInc.value || ctx.settings.YforceSbtPhases.value
-    super.isRunnable && (ctx.sbtCallback != null || forceRun)
+    super.isRunnable && (ctx.sbtCallback != null && ctx.sbtCallback.enabled() || forceRun)
   }
 
   // Check no needed. Does not transform trees
@@ -91,7 +93,7 @@ class ExtractDependencies extends Phase {
       } finally pw.close()
     }
 
-    if (ctx.sbtCallback != null) {
+    if (ctx.sbtCallback != null && ctx.sbtCallback.enabled()) {
       collector.usedNames.foreach {
         case (clazz, usedNames) =>
           val className = classNameAsString(clazz)
@@ -112,17 +114,17 @@ class ExtractDependencies extends Phase {
    */
   def recordDependency(dep: ClassDependency)(using Context): Unit = {
     val fromClassName = classNameAsString(dep.from)
-    val sourceFile = ctx.compilationUnit.source.file.file
+    val zincSourceFile = ctx.compilationUnit.source.underlyingZincFile
 
-    def binaryDependency(file: File, binaryClassName: String) =
-      ctx.sbtCallback.binaryDependency(file, binaryClassName, fromClassName, sourceFile, dep.context)
+    def binaryDependency(file: Path, binaryClassName: String) =
+      ctx.sbtCallback.binaryDependency(file, binaryClassName, fromClassName, zincSourceFile, dep.context)
 
     def processExternalDependency(depFile: AbstractFile, binaryClassName: String) = {
       depFile match {
         case ze: ZipArchive#Entry => // The dependency comes from a JAR
           ze.underlyingSource match
-            case Some(zip) if zip.file != null =>
-              binaryDependency(zip.file, binaryClassName)
+            case Some(zip) if zip.jpath != null =>
+              binaryDependency(zip.jpath, binaryClassName)
             case _ =>
         case pf: PlainFile => // The dependency comes from a class file
           // FIXME: pf.file is null for classfiles coming from the modulepath
@@ -131,15 +133,19 @@ class ExtractDependencies extends Phase {
           // java.io.File, this means that we cannot record dependencies coming
           // from the modulepath. For now this isn't a big deal since we only
           // support having the standard Java library on the modulepath.
-          if pf.file != null then
-            binaryDependency(pf.file, binaryClassName)
+          if pf.jpath != null then
+            binaryDependency(pf.jpath, binaryClassName)
         case _ =>
           internalError(s"Ignoring dependency $depFile of unknown class ${depFile.getClass}}", dep.from.srcPos)
       }
     }
 
     val depFile = dep.to.associatedFile
     if (depFile != null) {
+      def depIsSameSource =
+        val depVF: xsbti.VirtualFile | Null = ctx.zincVirtualFiles.uncheckedNN.get(depFile.absolutePath)
+        depVF != null && depVF.id() == zincSourceFile.id()
+
       // Cannot ignore inheritance relationship coming from the same source (see sbt/zinc#417)
       def allowLocal = dep.context == DependencyByInheritance || dep.context == LocalDependencyByInheritance
       val depClassFile =
@@ -148,7 +154,7 @@ class ExtractDependencies extends Phase {
       if (depClassFile != null) {
         // Dependency is external -- source is undefined
         processExternalDependency(depClassFile, dep.to.binaryClassName)
-      } else if (allowLocal || depFile.file != sourceFile) {
+      } else if (allowLocal || !depIsSameSource /* old: depFile.file != sourceFile.file */) {
         // We cannot ignore dependencies coming from the same source file because
         // the dependency info needs to propagate. See source-dependencies/trait-trait-211.
         val toClassName = classNameAsString(dep.to)

compiler/src/dotty/tools/dotc/util/SourceFile.scala

@@ -63,6 +63,33 @@ class SourceFile(val file: AbstractFile, computeContent: => Array[Char]) extends
   import SourceFile._
 
   private var myContent: Array[Char] | Null = null
+  private var myUnderlyingZincFile: xsbti.VirtualFile | Null = null
+
+  def underlyingZincFile(using Context): xsbti.VirtualFile =
+    val local = myUnderlyingZincFile
+    if local == null then
+      // usually without -sourcepath then the `underlying` will be set by Zinc.
+      val maybeUnderlying = file.underlying
+      val underlying0 =
+        if maybeUnderlying == null then
+          // When we have `-sourcepath` set then the file could come from the filesystem,
+          // rather than a zinc managed file, so then we need to check if we have a virtual file for it.
+          // TODO: we should consider in the future if there is a use case for sourcepath to possibly be
+          //       made of virtual files.
+          val fromLookup = ctx.zincVirtualFiles.uncheckedNN.get(file.absolutePath)
+          if fromLookup != null then
+            fromLookup
+          else
+            sys.error(s"no underlying file for ${file.absolutePath}, possible paths = ${ctx.zincVirtualFiles.keySet}")
+        else maybeUnderlying
+      if ctx.settings.YdebugVirtualFiles.value then
+        val isVirtual = !underlying0.isInstanceOf[xsbti.PathBasedFile]
+        println(s"found underlying zinc file ${underlying0.id} for ${file.absolutePath} [virtual = $isVirtual]")
+
+      myUnderlyingZincFile = underlying0
+      underlying0
+    else
+      local
 
   /** The contents of the original source file. Note that this can be empty, for example when
    * the source is read from Tasty. */

compiler/src/dotty/tools/io/AbstractFile.scala

@@ -118,6 +118,9 @@ abstract class AbstractFile extends Iterable[AbstractFile] {
   /** Returns the underlying Path if any and null otherwise. */
   def jpath: JPath
 
+  /** Overridden in sbt-bridge ZincPlainFile and ZincVirtualFile */
+  def underlying: xsbti.VirtualFile | Null = null
+
   /** An underlying source, if known.  Mostly, a zip/jar file. */
   def underlyingSource: Option[AbstractFile] = None
 

project/Build.scala

@@ -556,7 +556,7 @@ object Build {
       // get libraries onboard
       libraryDependencies ++= Seq(
         "org.scala-lang.modules" % "scala-asm" % "9.5.0-scala-1", // used by the backend
-        Dependencies.oldCompilerInterface, // we stick to the old version to avoid deprecation warnings
+        Dependencies.newCompilerInterface, // we stick to the old version to avoid deprecation warnings
         "org.jline" % "jline-reader" % "3.19.0",   // used by the REPL
         "org.jline" % "jline-terminal" % "3.19.0",
         "org.jline" % "jline-terminal-jna" % "3.19.0", // needed for Windows

project/Dependencies.scala

@@ -29,5 +29,5 @@ object Dependencies {
   )
 
   val newCompilerInterface = "org.scala-sbt" % "compiler-interface" % "1.8.0"
-  val oldCompilerInterface = "org.scala-sbt" % "compiler-interface" % "1.3.5"
+  // val oldCompilerInterface = "org.scala-sbt" % "compiler-interface" % "1.3.5"
 }

sbt-bridge/src/dotty/tools/xsbt/CompilerBridgeDriver.java

@@ -19,6 +19,7 @@
 
 import java.io.IOException;
 import java.util.Comparator;
+import java.util.HashMap;
 import java.util.Arrays;
 
 public class CompilerBridgeDriver extends Driver {
@@ -51,14 +52,40 @@ public boolean sourcesRequired() {
   }
 
   synchronized public void run(VirtualFile[] sources, AnalysisCallback callback, Logger log, Reporter delegate) {
-    DelegatingReporter reporter = new DelegatingReporter(delegate);
+    // convert sources to a HashMap from this.id to itself
+    HashMap<String, VirtualFile> sourcesMap = new HashMap<>();
+
+    VirtualFile[] sortedSources = new VirtualFile[sources.length];
+    System.arraycopy(sources, 0, sortedSources, 0, sources.length);
+    Arrays.sort(sortedSources, (x0, x1) -> x0.id().compareTo(x1.id()));
+
+    ListBuffer<AbstractFile> sourcesBuffer = new ListBuffer<>();
+
+    for (int i = 0; i < sources.length; i++) {
+      VirtualFile source = sortedSources[i];
+      AbstractFile abstractFile = asDottyFile(source);
+      sourcesBuffer.append(abstractFile);
+      sourcesMap.put(abstractFile.absolutePath(), source);
+    }
+
+    DelegatingReporter reporter = new DelegatingReporter(delegate, sourceFile -> {
+      String pathId = sourceFile.file().absolutePath();
+      VirtualFile source = sourcesMap.get(pathId);
+
+      if (source != null)
+        return source.id();
+      else
+        return pathId;
+    });
+
     try {
       log.debug(this::infoOnCachedCompiler);
 
       Contexts.Context initialCtx = initCtx()
         .fresh()
         .setReporter(reporter)
-        .setSbtCallback(callback);
+        .setSbtCallback(callback)
+        .setZincVirtualFiles(sourcesMap);
 
       Contexts.Context context = setup(args, initialCtx).map(t -> t._2).getOrElse(() -> initialCtx);
 
@@ -70,28 +97,13 @@ synchronized public void run(VirtualFile[] sources, AnalysisCallback callback, L
         log.debug(this::prettyPrintCompilationArguments);
         Compiler compiler = newCompiler(context);
 
-        VirtualFile[] sortedSources = new VirtualFile[sources.length];
-        System.arraycopy(sources, 0, sortedSources, 0, sources.length);
-        Arrays.sort(
-          sortedSources,
-          new Comparator<VirtualFile>() {
-            @Override
-            public int compare(VirtualFile x0, VirtualFile x1) {
-              return x0.id().compareTo(x1.id());
-            }
-          }
-        );
-
-        ListBuffer<AbstractFile> sourcesBuffer = new ListBuffer<>();
-        for (VirtualFile file: sortedSources)
-          sourcesBuffer.append(asDottyFile(file));
         doCompile(compiler, sourcesBuffer.toList(), context);
 
         for (xsbti.Problem problem: delegate.problems()) {
           callback.problem(problem.category(), problem.position(), problem.message(), problem.severity(),
             true);
         }
-      } else {  
+      } else {
         delegate.printSummary();
       }
 

sbt-bridge/src/dotty/tools/xsbt/DelegatingReporter.java

@@ -15,12 +15,16 @@
 import xsbti.Position;
 import xsbti.Severity;
 
+import java.util.function.*;
+
 final public class DelegatingReporter extends AbstractReporter {
   private xsbti.Reporter delegate;
+  private final Function<SourceFile, String> lookup;
 
-  public DelegatingReporter(xsbti.Reporter delegate) {
+  public DelegatingReporter(xsbti.Reporter delegate, Function<SourceFile, String> lookup) {
     super();
     this.delegate = delegate;
+    this.lookup = lookup;
   }
 
   public void dropDelegate() {
@@ -63,9 +67,9 @@ private static Severity severityOf(int level) {
     return severity;
   }
 
-  private static Position positionOf(SourcePosition pos) {
-    if (pos.exists()){
-      return new PositionBridge(pos, pos.source());
+  private Position positionOf(SourcePosition pos) {
+    if (pos.exists()) {
+      return new PositionBridge(pos, lookup.apply(pos.source()));
     } else {
       return PositionBridge.noPosition;
     }