Programmable Classpath to support Pickle caching, Build Pipelining

src/compiler/scala/tools/nsc/backend/JavaPlatform.scala

@@ -27,7 +27,7 @@ trait JavaPlatform extends Platform {
   private[nsc] var currentClassPath: Option[ClassPath] = None
 
   private[nsc] def classPath: ClassPath = {
-    if (currentClassPath.isEmpty) currentClassPath = Some(new PathResolver(settings).result)
+    if (currentClassPath.isEmpty) currentClassPath = Some(applyClassPathPlugins(new PathResolver(settings).result))
     currentClassPath.get
   }
 

src/compiler/scala/tools/nsc/backend/Platform.scala

@@ -13,7 +13,10 @@
 package scala.tools.nsc
 package backend
 
+import java.nio.ByteBuffer
+
 import io.AbstractFile
+import scala.tools.nsc.classpath.AggregateClassPath
 import scala.tools.nsc.util.ClassPath
 
 /** The platform dependent pieces of Global.
@@ -44,5 +47,57 @@ trait Platform {
    * a re-compile is triggered. On .NET by contrast classfiles always take precedence.
    */
   def needCompile(bin: AbstractFile, src: AbstractFile): Boolean
+
+  /**
+    * A class path plugin can modify the classpath before it is used by the compiler, and can
+    * customize the way that the compiler reads the contents of class files.
+    *
+    * Applications could include:
+    *
+    *   - Caching the ScalaSignature annotation contents, to avoid the cost of decompressing
+    *     and parsing the classfile, akin to the OpenJDK's .sig format for stripped class files.
+    *   - Starting a downstream compilation job immediately after the upstream job has completed
+    *     the pickler phase ("Build Pipelineing")
+    */
+  abstract class ClassPathPlugin {
+    def info(file: AbstractFile, clazz: ClassSymbol): Option[ClassfileInfo]
+    def parsed(file: AbstractFile, clazz: ClassSymbol, info: ClassfileInfo): Unit = ()
+    def modifyClassPath(classPath: Seq[ClassPath]): Seq[ClassPath] = classPath
+  }
+
+  /** A list of registered classpath plugins */
+  private var classPathPlugins: List[ClassPathPlugin] = Nil
+
+  protected final def applyClassPathPlugins(original: ClassPath): ClassPath = {
+    val entries = original match {
+      case AggregateClassPath(entries) => entries
+      case single => single :: Nil
+    }
+    val entries1 = classPathPlugins.foldLeft(entries) {
+      (entries, plugin) => plugin.modifyClassPath(entries)
+    }
+    AggregateClassPath(entries1)
+  }
+
+
+  /** Registers a new classpath plugin */
+  final def addClassPathPlugin(plugin: ClassPathPlugin): Unit = {
+    if (!classPathPlugins.contains(plugin))
+      classPathPlugins = plugin :: classPathPlugins
+  }
+  final def classFileInfo(file: AbstractFile, clazz: ClassSymbol): Option[ClassfileInfo] = if (classPathPlugins eq Nil) None else {
+    classPathPlugins.foldLeft(Option.empty[ClassfileInfo]) {
+      case (Some(info), _) => Some(info)
+      case (None, plugin) => plugin.info(file, clazz)
+    }
+  }
+  final def classFileInfoParsed(file: AbstractFile, clazz: ClassSymbol, info: ClassfileInfo): Unit = if (classPathPlugins eq Nil) None else {
+    classPathPlugins.foreach(_.parsed(file, clazz, info))
+  }
 }
 
+sealed abstract class ClassfileInfo {}
+final case class ClassBytes(data: () => ByteBuffer) extends ClassfileInfo
+final case class ScalaRawClass(className: String) extends ClassfileInfo
+final case class ScalaClass(className: String, pickle: () => ByteBuffer) extends ClassfileInfo
+

src/compiler/scala/tools/nsc/classpath/VirtualDirectoryClassPath.scala

@@ -35,7 +35,7 @@ case class VirtualDirectoryClassPath(dir: VirtualDirectory) extends ClassPath wi
   def isPackage(f: AbstractFile): Boolean = f.isPackage
 
   // mimic the behavior of the old nsc.util.DirectoryClassPath
-  def asURLs: Seq[URL] = Seq(new URL(dir.name))
+  def asURLs: Seq[URL] = Seq(new URL("file://_VIRTUAL_/" + dir.name))
   def asClassPathStrings: Seq[String] = Seq(dir.path)
 
   override def findClass(className: String): Option[ClassRepresentation] = findClassFile(className) map ClassFileEntryImpl

src/compiler/scala/tools/nsc/settings/AbsSettings.scala

@@ -25,7 +25,7 @@ trait AbsSettings extends scala.reflect.internal.settings.AbsSettings {
   protected def allSettings: scala.collection.Set[Setting]
 
   // settings minus internal usage settings
-  def visibleSettings = allSettings filterNot (_.isInternalOnly)
+  def visibleSettings = allSettings.iterator filterNot (_.isInternalOnly)
 
   // only settings which differ from default
   def userSetSettings = visibleSettings filterNot (_.isDefault)

src/compiler/scala/tools/nsc/settings/ScalaSettings.scala

@@ -243,6 +243,7 @@ trait ScalaSettings extends AbsScalaSettings
   val YcacheMacroClassLoader   = CachePolicy.setting("macro", "macros")
   val YpartialUnification = BooleanSetting ("-Ypartial-unification", "Enable partial unification in type constructor inference")
   val Yvirtpatmat     = BooleanSetting    ("-Yvirtpatmat", "Enable pattern matcher virtualization")
+  val Youtline        = BooleanSetting    ("-Youtline", "Don't compile method bodies. Use together with `-Ystop-afer:pickler to generate the pickled signatures for all source files.")
 
   val exposeEmptyPackage = BooleanSetting ("-Yexpose-empty-package", "Internal only: expose the empty package.").internalOnly()
   val Ydelambdafy        = ChoiceSetting  ("-Ydelambdafy", "strategy", "Strategy used for translating lambdas into JVM code.", List("inline", "method"), "method")

src/compiler/scala/tools/nsc/symtab/SymbolLoaders.scala

@@ -277,6 +277,8 @@ abstract class SymbolLoaders {
 
       val classPathEntries = classPath.list(packageName)
 
+      if (root.name.string_==("immutable"))
+        getClass
       if (!root.isRoot)
         for (entry <- classPathEntries.classesAndSources) initializeFromClassPath(root, entry)
       if (!root.isEmptyPackageClass) {

src/compiler/scala/tools/nsc/symtab/classfile/AbstractFileReader.scala

@@ -25,11 +25,8 @@ import scala.tools.nsc.io.AbstractFile
  * @author Philippe Altherr
  * @version 1.0, 23/03/2004
  */
-class AbstractFileReader(val file: AbstractFile) {
-
-  /** the buffer containing the file
-   */
-  val buf: Array[Byte] = file.toByteArray
+class AbstractFileReader(val file: AbstractFile, val buf: Array[Byte]) {
+  def this(file: AbstractFile) = this(file, file.toByteArray)
 
   /** the current input pointer
    */

src/compiler/scala/tools/nsc/symtab/classfile/ClassfileParser.scala

@@ -15,16 +15,18 @@ package tools.nsc
 package symtab
 package classfile
 
-import java.io.{ByteArrayInputStream, DataInputStream, File, IOException}
+import java.io._
 import java.lang.Integer.toHexString
+import java.nio.ByteBuffer
 
 import scala.collection.{immutable, mutable}
 import scala.collection.mutable.{ArrayBuffer, ListBuffer}
 import scala.annotation.switch
 import scala.reflect.internal.JavaAccFlags
 import scala.reflect.internal.pickling.{ByteCodecs, PickleBuffer}
-import scala.reflect.io.NoAbstractFile
+import scala.reflect.io.{NoAbstractFile, VirtualFile}
 import scala.reflect.internal.util.Collections._
+import scala.tools.nsc.backend.{ClassBytes, ScalaClass, ScalaRawClass}
 import scala.tools.nsc.util.ClassPath
 import scala.tools.nsc.io.AbstractFile
 import scala.util.control.NonFatal
@@ -152,14 +154,45 @@ abstract class ClassfileParser {
   def parse(file: AbstractFile, clazz: ClassSymbol, module: ModuleSymbol): Unit = {
     this.file = file
     pushBusy(clazz) {
-      this.in           = new AbstractFileReader(file)
       this.clazz        = clazz
       this.staticModule = module
       this.isScala      = false
 
-      parseHeader()
-      this.pool = newConstantPool
-      parseClass()
+      import loaders.platform._
+      classFileInfo(file, clazz) match {
+        case Some(info) =>
+          info match {
+            case ScalaRawClass(className) =>
+              isScalaRaw = true
+              currentClass = TermName(className)
+            case ScalaClass(className, pickle) =>
+              val pickle1 = pickle()
+              isScala = true
+              currentClass = TermName(className)
+              if (pickle1.hasArray) {
+                unpickler.unpickle(pickle1.array, pickle1.arrayOffset + pickle1.position(), clazz, staticModule, file.name)
+              } else {
+                val array = new Array[Byte](pickle1.remaining)
+                pickle1.get(array)
+                unpickler.unpickle(array, 0, clazz, staticModule, file.name)
+              }
+            case ClassBytes(data) =>
+              val data1 = data()
+              val array = new Array[Byte](data1.remaining)
+              data1.get(array)
+              this.in = new AbstractFileReader(file, array)
+              parseHeader()
+              this.pool = newConstantPool
+              parseClass()
+          }
+        case None =>
+          this.in = new AbstractFileReader(file)
+          parseHeader()
+          this.pool = newConstantPool
+          parseClass()
+          if (!(isScala || isScalaRaw))
+            loaders.platform.classFileInfoParsed(file, clazz, ClassBytes(() => ByteBuffer.wrap(in.buf)))
+      }
     }
   }
 
@@ -441,6 +474,15 @@ abstract class ClassfileParser {
       lookupClass(name)
   }
 
+  // TODO: remove after the next 2.13 milestone
+  // A bug in the backend caused classes ending in `$` do get only a Scala marker attribute
+  // instead of a ScalaSig and a Signature annotaiton. This went unnoticed because isScalaRaw
+  // classes were parsed like Java classes. The below covers the cases in the std lib.
+  private def isNothingOrNull = {
+    val n = clazz.fullName.toString
+    n == "scala.runtime.Nothing$" || n == "scala.runtime.Null$"
+  }
+
   def parseClass() {
     val jflags   = readClassFlags()
     val sflags   = jflags.toScalaFlags
@@ -890,8 +932,8 @@ abstract class ClassfileParser {
               case Some(san: AnnotationInfo) =>
                 val bytes =
                   san.assocs.find({ _._1 == nme.bytes }).get._2.asInstanceOf[ScalaSigBytes].bytes
-
                 unpickler.unpickle(bytes, 0, clazz, staticModule, in.file.name)
+                loaders.platform.classFileInfoParsed(file, clazz, ScalaClass(this.currentClass.toString, () => ByteBuffer.wrap(bytes)))
               case None =>
                 throw new RuntimeException("Scala class file does not contain Scala annotation")
             }
@@ -1216,6 +1258,7 @@ abstract class ClassfileParser {
           in.skip(attrLen)
         case tpnme.ScalaATTR =>
           isScalaRaw = true
+          loaders.platform.classFileInfoParsed(file, clazz, ScalaRawClass(this.currentClass.toString))
         case tpnme.InnerClassesATTR if !isScala =>
           val entries = u2
           for (i <- 0 until entries) {

src/compiler/scala/tools/nsc/typechecker/Analyzer.scala

@@ -112,11 +112,13 @@ trait Analyzer extends AnyRef
         try {
           val typer = newTyper(rootContext(unit))
           unit.body = typer.typed(unit.body)
-          for (workItem <- unit.toCheck) workItem()
-          if (settings.warnUnusedImport)
-            warnUnusedImports(unit)
-          if (settings.warnUnused.isSetByUser)
-            new checkUnused(typer).apply(unit)
+          if (!settings.Youtline.value) {
+            for (workItem <- unit.toCheck) workItem()
+            if (settings.warnUnusedImport)
+              warnUnusedImports(unit)
+            if (settings.warnUnused.isSetByUser)
+              new checkUnused(typer).apply(unit)
+          }
         }
         finally {
           unit.toCheck.clear()

src/compiler/scala/tools/nsc/typechecker/Typers.scala

@@ -2057,8 +2057,9 @@ trait Typers extends Adaptations with Tags with TypersTracking with PatternTyper
     }
 
     // use typedValDef instead. this version is called after creating a new context for the ValDef
-    private def typedValDefImpl(vdef: ValDef) = {
+    private def typedValDefImpl(vdef: ValDef): ValDef = {
       val sym = vdef.symbol.initialize
+
       val typedMods = if (nme.isLocalName(sym.name) && sym.isPrivateThis && !vdef.mods.isPrivateLocal) {
         // scala/bug#10009 This tree has been given a field symbol by `enterGetterSetter`, patch up the
         // modifiers accordingly so that we can survive resetAttrs and retypechecking.
@@ -5845,7 +5846,7 @@ trait Typers extends Adaptations with Tags with TypersTracking with PatternTyper
     final def transformedOrTyped(tree: Tree, mode: Mode, pt: Type): Tree = {
       lookupTransformed(tree) match {
         case Some(tree1) => tree1
-        case _           => typed(tree, mode, pt)
+        case _           => if (settings.Youtline.value) EmptyTree else typed(tree, mode, pt)
       }
     }
     final def lookupTransformed(tree: Tree): Option[Tree] =

src/reflect/scala/reflect/internal/Symbols.scala

@@ -1978,7 +1978,7 @@ trait Symbols extends api.Symbols { self: SymbolTable =>
         var alts0: List[Symbol] = alternatives
         var alts1: List[Symbol] = Nil
 
-        while (alts0.nonEmpty) {
+        while (!alts0.isEmpty) {
           if (cond(alts0.head))
             alts1 ::= alts0.head
           else

src/reflect/scala/reflect/internal/pickling/UnPickler.scala

@@ -249,6 +249,9 @@ abstract class UnPickler {
           else NoSymbol
         }
 
+        if (owner == definitions.ScalaPackageClass && name == tpnme.AnyRef)
+          return definitions.AnyRefClass
+
         // (1) Try name.
         localDummy orElse fromName(name) orElse {
           // (2) Try with expanded name.  Can happen if references to private

src/repl/scala/tools/nsc/interpreter/ILoop.scala

@@ -363,7 +363,8 @@ class ILoop(in0: Option[BufferedReader], protected val out: JPrintWriter) extend
   }
 
   private def changeSettings(line: String): Result = {
-    def showSettings() = for (s <- settings.userSetSettings.toSeq.sorted) echo(s.toString)
+    val settings1 = settings
+    def showSettings() = for (s <- settings1.userSetSettings.toSeq.sorted) echo(s.toString)
     if (line.isEmpty) showSettings() else { updateSettings(line) ; () }
   }
   private def updateSettings(line: String) = {

test/junit/scala/tools/nsc/classpath/ClassPluginTest.scala

@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2018 Lightbend. All rights reserved.
+ */
+package scala.tools.nsc.classpath
+
+import java.nio.ByteBuffer
+
+import org.junit.Assert.assertEquals
+import org.junit.Test
+import org.junit.runner.RunWith
+import org.junit.runners.JUnit4
+
+import scala.reflect.io.VirtualDirectory
+import scala.tools.nsc.backend.{ClassfileInfo, ScalaClass}
+import scala.tools.nsc.io.AbstractFile
+import scala.tools.nsc.symtab.SymbolTableForUnitTesting
+import scala.tools.nsc.util.ClassPath
+import scala.tools.testing.BytecodeTesting
+import scala.tools.testing.BytecodeTesting.makeSourceFile
+
+@RunWith(classOf[JUnit4])
+class ClassPluginTest extends BytecodeTesting {
+  // We use this.compiler to generate Scala pickles...
+  override def compilerArgs = "-Ystop-after:pickler"
+
+  // ... and this one to read them with a ClassPathPlugin
+  object symbolTable extends SymbolTableForUnitTesting {
+    val fakeClasses = Map(
+      "fake.C" -> ScalaClass("fake.C", pickleOf("package fake; class C { def foo = 42 }"))
+    )
+    private val fakes = new VirtualDirectory("fakes", None)
+    fakes.subdirectoryNamed("fake").fileNamed("C.class")
+
+    lazy val classpathPlugin = new platform.ClassPathPlugin {
+      override def modifyClassPath(classPath: Seq[ClassPath]): Seq[ClassPath] = {
+        // Add a classpath entry with the fake/C.class
+        VirtualDirectoryClassPath(fakes) +: classPath
+      }
+
+      override def info(file: AbstractFile, clazz: ClassSymbol): Option[ClassfileInfo] =
+        fakeClasses.get(clazz.fullNameString)
+    }
+    this.platform.addClassPathPlugin(classpathPlugin)
+  }
+
+  @Test def classPathPluginTest(): Unit = {
+    import symbolTable._
+    val CClass = rootMirror.getRequiredClass("fake.C")
+    val C_tpe = CClass.info
+    assertEquals("def foo: Int", definitions.fullyInitializeSymbol(C_tpe.decl(TermName("foo"))).defString)
+  }
+
+  private def pickleOf(code: String): ByteBuffer = {
+    import compiler._
+    val run = newRun
+    run.compileSources(makeSourceFile(code, "unitTestSource.scala") :: Nil)
+    val pickle = run.symData.toList.head._2
+    ByteBuffer.wrap(pickle.bytes, 0, pickle.writeIndex)
+  }
+}

test/junit/scala/tools/nsc/symtab/SymbolTableForUnitTesting.scala

@@ -36,7 +36,7 @@ class SymbolTableForUnitTesting extends SymbolTable {
 
     def platformPhases: List[SubComponent] = Nil
 
-    private[nsc] lazy val classPath: ClassPath = new PathResolver(settings).result
+    private[nsc] lazy val classPath: ClassPath = applyClassPathPlugins(new PathResolver(settings).result)
 
     def isMaybeBoxed(sym: Symbol): Boolean = ???
     def needCompile(bin: AbstractFile, src: AbstractFile): Boolean = ???