Loading symbols from TASTy files directly

Author: nicolasstucki

compiler/src/dotty/tools/backend/jvm/PostProcessor.scala

@@ -23,6 +23,12 @@ class PostProcessor(val frontendAccess: PostProcessorFrontendAccess, val bTypes:
 
   def postProcessAndSendToDisk(generatedDefs: GeneratedDefs): Unit = {
     val GeneratedDefs(classes, tasty) = generatedDefs
+    if !ctx.settings.YoutputOnlyTasty.value then
+      postProcessClassesAndSendToDisk(classes)
+    postProcessTastyAndSendToDisk(tasty)
+  }
+
+  private def postProcessClassesAndSendToDisk(classes: List[GeneratedClass]): Unit = {
     for (GeneratedClass(classNode, sourceFile, isArtifact, onFileCreated) <- classes) {
       val bytes =
         try
@@ -46,8 +52,10 @@ class PostProcessor(val frontendAccess: PostProcessorFrontendAccess, val bTypes:
         if clsFile != null then onFileCreated(clsFile)
       }
     }
+  }
 
-    for (GeneratedTasty(classNode, binaryGen) <- tasty){
+  private def postProcessTastyAndSendToDisk(tasty: List[GeneratedTasty]): Unit = {
+    for (GeneratedTasty(classNode, binaryGen) <- tasty) {
       classfileWriter.writeTasty(classNode.name.nn, binaryGen())
     }
   }

compiler/src/dotty/tools/dotc/classpath/DirectoryClassPath.scala

@@ -278,15 +278,17 @@ case class DirectoryClassPath(dir: JFile) extends JFileDirectoryLookup[ClassFile
 
   def findClassFile(className: String): Option[AbstractFile] = {
     val relativePath = FileUtils.dirPath(className)
-    val classFile = new JFile(dir, relativePath + ".class")
-    if (classFile.exists) {
-      Some(classFile.toPath.toPlainFile)
-    }
-    else None
+    val tastyFile = new JFile(dir, relativePath + ".tasty")
+    if tastyFile.exists then Some(tastyFile.toPath.toPlainFile)
+    else
+      val classFile = new JFile(dir, relativePath + ".class")
+      if classFile.exists then  Some(classFile.toPath.toPlainFile)
+      else None
   }
 
   protected def createFileEntry(file: AbstractFile): ClassFileEntryImpl = ClassFileEntryImpl(file)
-  protected def isMatchingFile(f: JFile): Boolean = f.isClass
+  protected def isMatchingFile(f: JFile): Boolean =
+    f.isTasty || (f.isClass && f.classToTasty.isEmpty)
 
   private[dotty] def classes(inPackage: PackageName): Seq[ClassFileEntry] = files(inPackage)
 }

compiler/src/dotty/tools/dotc/classpath/FileUtils.scala

@@ -20,6 +20,10 @@ object FileUtils {
     def isClass: Boolean = !file.isDirectory && file.hasExtension("class") && !file.name.endsWith("$class.class")
       // FIXME: drop last condition when we stop being compatible with Scala 2.11
 
+    def isTasty: Boolean = !file.isDirectory && file.hasExtension("tasty")
+
+    def isScalaBinary: Boolean = file.isClass || file.isTasty
+
     def isScalaOrJavaSource: Boolean = !file.isDirectory && (file.hasExtension("scala") || file.hasExtension("java"))
 
     // TODO do we need to check also other files using ZipMagicNumber like in scala.tools.nsc.io.Jar.isJarOrZip?
@@ -30,17 +34,34 @@ object FileUtils {
      * and returning given default value in other case
      */
     def toURLs(default: => Seq[URL] = Seq.empty): Seq[URL] = if (file.file == null) default else Seq(file.toURL)
+
+    /** Returns the tasty file associated with this class file */
+    def classToTasty: Option[AbstractFile] =
+      assert(file.isClass, s"non-class: $file")
+      val tastyName = classNameToTasty(file.name)
+      Option(file.resolveSibling(tastyName))
   }
 
   extension (file: JFile) {
     def isPackage: Boolean = file.isDirectory && mayBeValidPackage(file.getName)
 
-    def isClass: Boolean = file.isFile && file.getName.endsWith(".class") && !file.getName.endsWith("$class.class")
-      // FIXME: drop last condition when we stop being compatible with Scala 2.11
+    def isClass: Boolean = file.isFile && file.getName.endsWith(SUFFIX_CLASS) && !file.getName.endsWith("$class.class")
+    // FIXME: drop last condition when we stop being compatible with Scala 2.11
+
+    def isTasty: Boolean = file.isFile && file.getName.endsWith(SUFFIX_TASTY)
+
+    /** Returns the tasty file associated with this class file */
+    def classToTasty: Option[JFile] =
+      assert(file.isClass, s"non-class: $file")
+      val tastyName = classNameToTasty(file.getName.stripSuffix(".class"))
+      val tastyPath = file.toPath.resolveSibling(tastyName)
+      if java.nio.file.Files.exists(tastyPath) then Some(tastyPath.toFile) else None
+
   }
 
   private val SUFFIX_CLASS = ".class"
   private val SUFFIX_SCALA = ".scala"
+  private val SUFFIX_TASTY = ".tasty"
   private val SUFFIX_JAVA = ".java"
   private val SUFFIX_SIG = ".sig"
 
@@ -81,4 +102,15 @@ object FileUtils {
   def mkFileFilter(f: JFile => Boolean): FileFilter = new FileFilter {
     def accept(pathname: JFile): Boolean = f(pathname)
   }
+
+  /** Transforms a .class file name to a .tasty file name */
+  private def classNameToTasty(fileName: String): String =
+    val classOrModuleName = fileName.stripSuffix(".class")
+    val className =
+      if classOrModuleName.endsWith("$")
+        && classOrModuleName != "Null$" // scala.runtime.Null$
+        && classOrModuleName != "Nothing$" // scala.runtime.Nothing$
+      then classOrModuleName.stripSuffix("$")
+      else classOrModuleName
+    className + SUFFIX_TASTY
 }

compiler/src/dotty/tools/dotc/classpath/VirtualDirectoryClassPath.scala

@@ -41,12 +41,17 @@ case class VirtualDirectoryClassPath(dir: VirtualDirectory) extends ClassPath wi
   override def findClass(className: String): Option[ClassRepresentation] = findClassFile(className) map ClassFileEntryImpl.apply
 
   def findClassFile(className: String): Option[AbstractFile] = {
-    val relativePath = FileUtils.dirPath(className) + ".class"
-    Option(lookupPath(dir)(relativePath.split(java.io.File.separator).toIndexedSeq, directory = false))
+    val pathSeq = FileUtils.dirPath(className).split(java.io.File.separator)
+    val parentDir = lookupPath(dir)(pathSeq.init.toSeq, directory = true)
+    if parentDir == null then return None
+    else
+      Option(lookupPath(parentDir)(pathSeq.last + ".tasty" :: Nil, directory = false))
+        .orElse(Option(lookupPath(parentDir)(pathSeq.last + ".class" :: Nil, directory = false)))
   }
 
   private[dotty] def classes(inPackage: PackageName): Seq[ClassFileEntry] = files(inPackage)
 
   protected def createFileEntry(file: AbstractFile): ClassFileEntryImpl = ClassFileEntryImpl(file)
-  protected def isMatchingFile(f: AbstractFile): Boolean = f.isClass
+  protected def isMatchingFile(f: AbstractFile): Boolean =
+    f.isTasty || (f.isClass && f.classToTasty.isEmpty)
 }

compiler/src/dotty/tools/dotc/classpath/ZipAndJarFileLookupFactory.scala

@@ -44,21 +44,21 @@ object ZipAndJarClassPathFactory extends ZipAndJarFileLookupFactory {
     extends ZipArchiveFileLookup[ClassFileEntryImpl]
     with NoSourcePaths {
 
-    override def findClassFile(className: String): Option[AbstractFile] = {
-      val (pkg, simpleClassName) = PackageNameUtils.separatePkgAndClassNames(className)
-      file(PackageName(pkg), simpleClassName + ".class").map(_.file)
-    }
+    override def findClassFile(className: String): Option[AbstractFile] =
+      findClass(className).map(_.file)
 
     // This method is performance sensitive as it is used by SBT's ExtractDependencies phase.
-    override def findClass(className: String): Option[ClassRepresentation] = {
+    override def findClass(className: String): Option[ClassFileEntryImpl] = {
       val (pkg, simpleClassName) = PackageNameUtils.separatePkgAndClassNames(className)
-      file(PackageName(pkg), simpleClassName + ".class")
+      val binaries = files(PackageName(pkg), simpleClassName + ".tasty", simpleClassName + ".class")
+      binaries.find(_.file.isTasty).orElse(binaries.find(_.file.isClass))
     }
 
     override private[dotty] def classes(inPackage: PackageName): Seq[ClassFileEntry] = files(inPackage)
 
     override protected def createFileEntry(file: FileZipArchive#Entry): ClassFileEntryImpl = ClassFileEntryImpl(file)
-    override protected def isRequiredFileType(file: AbstractFile): Boolean = file.isClass
+    override protected def isRequiredFileType(file: AbstractFile): Boolean =
+      file.isTasty || (file.isClass && file.classToTasty.isEmpty)
   }
 
   /**

compiler/src/dotty/tools/dotc/classpath/ZipArchiveFileLookup.scala

@@ -43,6 +43,15 @@ trait ZipArchiveFileLookup[FileEntryType <: ClassRepresentation] extends Efficie
     }
     yield createFileEntry(entry)
 
+  protected def files(inPackage: PackageName, names: String*): Seq[FileEntryType] =
+    for {
+      dirEntry <- findDirEntry(inPackage).toSeq
+      name <- names
+      entry <- Option(dirEntry.lookupName(name, directory = false))
+      if isRequiredFileType(entry)
+    }
+    yield createFileEntry(entry)
+
   protected def file(inPackage: PackageName, name: String): Option[FileEntryType] =
     for {
       dirEntry <- findDirEntry(inPackage)

compiler/src/dotty/tools/dotc/config/JavaPlatform.scala

@@ -66,4 +66,7 @@ class JavaPlatform extends Platform {
 
   def newClassLoader(bin: AbstractFile)(using Context): SymbolLoader =
     new ClassfileLoader(bin)
+
+  def newTastyLoader(bin: AbstractFile)(using Context): SymbolLoader =
+    new TastyLoader(bin)
 }

compiler/src/dotty/tools/dotc/config/Platform.scala

@@ -36,6 +36,9 @@ abstract class Platform {
   /** Create a new class loader to load class file `bin` */
   def newClassLoader(bin: AbstractFile)(using Context): SymbolLoader
 
+  /** Create a new TASTy loader to load class file `bin` */
+  def newTastyLoader(bin: AbstractFile)(using Context): SymbolLoader
+
   /** The given symbol is a method with the right name and signature to be a runnable program. */
   def isMainMethod(sym: Symbol)(using Context): Boolean
 

compiler/src/dotty/tools/dotc/config/ScalaSettings.scala

@@ -369,6 +369,7 @@ private sealed trait YSettings:
   val YnoExperimental: Setting[Boolean] = BooleanSetting("-Yno-experimental", "Disable experimental language features.")
   val YlegacyLazyVals: Setting[Boolean] = BooleanSetting("-Ylegacy-lazy-vals", "Use legacy (pre 3.3.0) implementation of lazy vals.")
   val Yscala2Stdlib: Setting[Boolean] = BooleanSetting("-Yscala2-stdlib", "Used when compiling the Scala 2 standard library.")
+  val YoutputOnlyTasty: Setting[Boolean] = BooleanSetting("-Youtput-only-tasty", "Used to only generate the TASTy file without the classfiles")
 
   val YprofileEnabled: Setting[Boolean] = BooleanSetting("-Yprofile-enabled", "Enable profiling.")
   val YprofileDestination: Setting[String] = StringSetting("-Yprofile-destination", "file", "Where to send profiling output - specify a file, default is to the console.", "")

compiler/src/dotty/tools/dotc/core/SymbolLoaders.scala

@@ -7,6 +7,7 @@ import java.nio.channels.ClosedByInterruptException
 
 import scala.util.control.NonFatal
 
+import dotty.tools.dotc.classpath.FileUtils.isTasty
 import dotty.tools.io.{ ClassPath, ClassRepresentation, AbstractFile }
 import dotty.tools.backend.jvm.DottyBackendInterface.symExtensions
 
@@ -192,10 +193,13 @@ object SymbolLoaders {
         if (ctx.settings.verbose.value) report.inform("[symloader] picked up newer source file for " + src.path)
         enterToplevelsFromSource(owner, nameOf(classRep), src)
       case (None, Some(src)) =>
-        if (ctx.settings.verbose.value) report.inform("[symloader] no class, picked up source file for " + src.path)
+        if (ctx.settings.verbose.value) report.inform("[symloader] no class or tasty, picked up source file for " + src.path)
         enterToplevelsFromSource(owner, nameOf(classRep), src)
       case (Some(bin), _) =>
-        enterClassAndModule(owner, nameOf(classRep), ctx.platform.newClassLoader(bin))
+        val completer =
+          if bin.isTasty then ctx.platform.newTastyLoader(bin)
+          else ctx.platform.newClassLoader(bin)
+        enterClassAndModule(owner, nameOf(classRep), completer)
     }
 
   def needCompile(bin: AbstractFile, src: AbstractFile): Boolean =
@@ -404,20 +408,27 @@ class ClassfileLoader(val classfile: AbstractFile) extends SymbolLoader {
   def description(using Context): String = "class file " + classfile.toString
 
   override def doComplete(root: SymDenotation)(using Context): Unit =
-    load(root)
-
-  def load(root: SymDenotation)(using Context): Unit = {
     val (classRoot, moduleRoot) = rootDenots(root.asClass)
     val classfileParser = new ClassfileParser(classfile, classRoot, moduleRoot)(ctx)
-    val result = classfileParser.run()
-    if (mayLoadTreesFromTasty)
-      result match {
-        case Some(unpickler: tasty.DottyUnpickler) =>
-          classRoot.classSymbol.rootTreeOrProvider = unpickler
-          moduleRoot.classSymbol.rootTreeOrProvider = unpickler
-        case _ =>
-      }
-  }
+    classfileParser.run()
+}
+
+class TastyLoader(val tastyFile: AbstractFile) extends SymbolLoader {
+
+  override def sourceFileOrNull: AbstractFile | Null = tastyFile
+
+  def description(using Context): String = "TASTy file " + tastyFile.toString
+
+  override def doComplete(root: SymDenotation)(using Context): Unit =
+    val (classRoot, moduleRoot) = rootDenots(root.asClass)
+    val unpickler =
+      val tastyBytes = tastyFile.toByteArray
+      new tasty.DottyUnpickler(tastyBytes)
+    unpickler.enter(roots = Set(classRoot, moduleRoot, moduleRoot.sourceModule))(using ctx.withSource(util.NoSource))
+    if mayLoadTreesFromTasty then
+      classRoot.classSymbol.rootTreeOrProvider = unpickler
+      moduleRoot.classSymbol.rootTreeOrProvider = unpickler
+    // TODO check TASTy UUID matches classfile
 
   private def mayLoadTreesFromTasty(using Context): Boolean =
     ctx.settings.YretainTrees.value || ctx.settings.fromTasty.value

compiler/src/dotty/tools/dotc/core/Symbols.scala

@@ -31,6 +31,7 @@ import io.AbstractFile
 import util.{SourceFile, NoSource, Property, SourcePosition, SrcPos, EqHashMap}
 import scala.annotation.internal.sharable
 import config.Printers.typr
+import dotty.tools.dotc.classpath.FileUtils.isScalaBinary
 
 object Symbols {
 
@@ -151,7 +152,7 @@ object Symbols {
       * symbols defined by the user in a prior run of the REPL, that are still valid.
       */
     final def isDefinedInSource(using Context): Boolean =
-      span.exists && isValidInCurrentRun && associatedFileMatches(_.extension != "class")
+      span.exists && isValidInCurrentRun && associatedFileMatches(!_.isScalaBinary)
 
     /** Is symbol valid in current run? */
     final def isValidInCurrentRun(using Context): Boolean =
@@ -272,7 +273,7 @@ object Symbols {
     /** The class file from which this class was generated, null if not applicable. */
     final def binaryFile(using Context): AbstractFile | Null = {
       val file = associatedFile
-      if (file != null && file.extension == "class") file else null
+      if file != null && file.isScalaBinary then file else null
     }
 
     /** A trap to avoid calling x.symbol on something that is already a symbol.
@@ -285,7 +286,7 @@ object Symbols {
 
     final def source(using Context): SourceFile = {
       def valid(src: SourceFile): SourceFile =
-        if (src.exists && src.file.extension != "class") src
+        if (src.exists && !src.file.isScalaBinary) src
         else NoSource
 
       if (!denot.exists) NoSource
@@ -463,7 +464,7 @@ object Symbols {
       if !mySource.exists && !denot.is(Package) then
         // this allows sources to be added in annotations after `sourceOfClass` is first called
         val file = associatedFile
-        if file != null && file.extension != "class" then
+        if file != null && !file.isScalaBinary then
           mySource = ctx.getSource(file)
         else
           mySource = defn.patchSource(this)