Skip to content

Commit 3e52000

Browse files
retronymretronym
committed
Classpath infrastracture to support pipelining, caching.
1 parent d904ee7 commit 3e52000

File tree

15 files changed

+655
-24
lines changed

15 files changed

+655
-24
lines changed

‎src/compiler/scala/tools/nsc/PipelineMain.scala

Lines changed: 456 additions & 0 deletions
Large diffs are not rendered by default.

‎src/compiler/scala/tools/nsc/backend/JavaPlatform.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ trait JavaPlatform extends Platform {
2727
private[nsc] var currentClassPath: Option[ClassPath] = None
2828

2929
private[nsc] def classPath: ClassPath = {
30-
if (currentClassPath.isEmpty) currentClassPath = Some(new PathResolver(settings).result)
30+
if (currentClassPath.isEmpty) currentClassPath = Some(applyClassPathPlugins(new PathResolver(settings).result))
3131
currentClassPath.get
3232
}
3333

‎src/compiler/scala/tools/nsc/backend/Platform.scala

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,10 @@
1313
package scala.tools.nsc
1414
package backend
1515

16+
import java.nio.ByteBuffer
17+
1618
import io.AbstractFile
19+
import scala.tools.nsc.classpath.AggregateClassPath
1720
import scala.tools.nsc.util.ClassPath
1821

1922
/** The platform dependent pieces of Global.
@@ -44,5 +47,57 @@ trait Platform {
4447
* a re-compile is triggered. On .NET by contrast classfiles always take precedence.
4548
*/
4649
def needCompile(bin: AbstractFile, src: AbstractFile): Boolean
50+
51+
/**
52+
* A class path plugin can modify the classpath before it is used by the compiler, and can
53+
* customize the way that the compiler reads the contents of class files.
54+
*
55+
* Applications could include:
56+
*
57+
* - Caching the ScalaSignature annotation contents, to avoid the cost of decompressing
58+
* and parsing the classfile, akin to the OpenJDK's .sig format for stripped class files.
59+
* - Starting a downstream compilation job immediately after the upstream job has completed
60+
* the pickler phase ("Build Pipelineing")
61+
*/
62+
abstract class ClassPathPlugin {
63+
def info(file: AbstractFile, clazz: ClassSymbol): Option[ClassfileInfo]
64+
def parsed(file: AbstractFile, clazz: ClassSymbol, info: ClassfileInfo): Unit = ()
65+
def modifyClassPath(classPath: Seq[ClassPath]): Seq[ClassPath] = classPath
66+
}
67+
68+
/** A list of registered classpath plugins */
69+
private var classPathPlugins: List[ClassPathPlugin] = Nil
70+
71+
protected final def applyClassPathPlugins(original: ClassPath): ClassPath = {
72+
val entries = original match {
73+
case AggregateClassPath(entries) => entries
74+
case single => single :: Nil
75+
}
76+
val entries1 = classPathPlugins.foldLeft(entries) {
77+
(entries, plugin) => plugin.modifyClassPath(entries)
78+
}
79+
AggregateClassPath(entries1)
80+
}
81+
82+
83+
/** Registers a new classpath plugin */
84+
final def addClassPathPlugin(plugin: ClassPathPlugin): Unit = {
85+
if (!classPathPlugins.contains(plugin))
86+
classPathPlugins = plugin :: classPathPlugins
87+
}
88+
final def classFileInfo(file: AbstractFile, clazz: ClassSymbol): Option[ClassfileInfo] = if (classPathPlugins eq Nil) None else {
89+
classPathPlugins.foldLeft(Option.empty[ClassfileInfo]) {
90+
case (Some(info), _) => Some(info)
91+
case (None, plugin) => plugin.info(file, clazz)
92+
}
93+
}
94+
final def classFileInfoParsed(file: AbstractFile, clazz: ClassSymbol, info: ClassfileInfo): Unit = if (classPathPlugins eq Nil) None else {
95+
classPathPlugins.foreach(_.parsed(file, clazz, info))
96+
}
4797
}
4898

99+
sealed abstract class ClassfileInfo {}
100+
final case class ClassBytes(data: ByteBuffer) extends ClassfileInfo
101+
final case class ScalaRawClass(className: String) extends ClassfileInfo
102+
final case class ScalaClass(className: String, pickle: ByteBuffer) extends ClassfileInfo
103+

‎src/compiler/scala/tools/nsc/classpath/VirtualDirectoryClassPath.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ case class VirtualDirectoryClassPath(dir: VirtualDirectory) extends ClassPath wi
3535
def isPackage(f: AbstractFile): Boolean = f.isPackage
3636

3737
// mimic the behavior of the old nsc.util.DirectoryClassPath
38-
def asURLs: Seq[URL] = Seq(new URL(dir.name))
38+
def asURLs: Seq[URL] = Seq(new URL("file://_VIRTUAL_/" + dir.name))
3939
def asClassPathStrings: Seq[String] = Seq(dir.path)
4040

4141
override def findClass(className: String): Option[ClassRepresentation] = findClassFile(className) map ClassFileEntryImpl

‎src/compiler/scala/tools/nsc/settings/AbsSettings.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ trait AbsSettings extends scala.reflect.internal.settings.AbsSettings {
2525
protected def allSettings: scala.collection.Set[Setting]
2626

2727
// settings minus internal usage settings
28-
def visibleSettings = allSettings filterNot (_.isInternalOnly)
28+
def visibleSettings = allSettings.iterator filterNot (_.isInternalOnly)
2929

3030
// only settings which differ from default
3131
def userSetSettings = visibleSettings filterNot (_.isDefault)

‎src/compiler/scala/tools/nsc/settings/ScalaSettings.scala

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -243,6 +243,7 @@ trait ScalaSettings extends AbsScalaSettings
243243
val YcacheMacroClassLoader = CachePolicy.setting("macro", "macros")
244244
val YpartialUnification = BooleanSetting ("-Ypartial-unification", "Enable partial unification in type constructor inference")
245245
val Yvirtpatmat = BooleanSetting ("-Yvirtpatmat", "Enable pattern matcher virtualization")
246+
val Youtline = BooleanSetting ("-Youtline", "Don't compile method bodies. Use together with `-Ystop-afer:pickler to generate the pickled signatures for all source files.")
246247

247248
val exposeEmptyPackage = BooleanSetting ("-Yexpose-empty-package", "Internal only: expose the empty package.").internalOnly()
248249
val Ydelambdafy = ChoiceSetting ("-Ydelambdafy", "strategy", "Strategy used for translating lambdas into JVM code.", List("inline", "method"), "method")

‎src/compiler/scala/tools/nsc/symtab/SymbolLoaders.scala

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -277,6 +277,8 @@ abstract class SymbolLoaders {
277277

278278
val classPathEntries = classPath.list(packageName)
279279

280+
if (root.name.string_==("immutable"))
281+
getClass
280282
if (!root.isRoot)
281283
for (entry <- classPathEntries.classesAndSources) initializeFromClassPath(root, entry)
282284
if (!root.isEmptyPackageClass) {

‎src/compiler/scala/tools/nsc/symtab/classfile/AbstractFileReader.scala

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -25,11 +25,8 @@ import scala.tools.nsc.io.AbstractFile
2525
* @author Philippe Altherr
2626
* @version 1.0, 23/03/2004
2727
*/
28-
class AbstractFileReader(val file: AbstractFile) {
29-
30-
/** the buffer containing the file
31-
*/
32-
val buf: Array[Byte] = file.toByteArray
28+
class AbstractFileReader(val file: AbstractFile, val buf: Array[Byte]) {
29+
def this(file: AbstractFile) = this(file, file.toByteArray)
3330

3431
/** the current input pointer
3532
*/

‎src/compiler/scala/tools/nsc/symtab/classfile/ClassfileParser.scala

Lines changed: 61 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,16 +15,18 @@ package tools.nsc
1515
package symtab
1616
package classfile
1717

18-
import java.io.{ByteArrayInputStream, DataInputStream, File, IOException}
18+
import java.io._
1919
import java.lang.Integer.toHexString
20+
import java.nio.ByteBuffer
2021

2122
import scala.collection.{immutable, mutable}
2223
import scala.collection.mutable.{ArrayBuffer, ListBuffer}
2324
import scala.annotation.switch
2425
import scala.reflect.internal.JavaAccFlags
2526
import scala.reflect.internal.pickling.{ByteCodecs, PickleBuffer}
26-
import scala.reflect.io.NoAbstractFile
27+
import scala.reflect.io.{NoAbstractFile, VirtualFile}
2728
import scala.reflect.internal.util.Collections._
29+
import scala.tools.nsc.backend.{ClassBytes, ScalaClass, ScalaRawClass}
2830
import scala.tools.nsc.util.ClassPath
2931
import scala.tools.nsc.io.AbstractFile
3032
import scala.util.control.NonFatal
@@ -152,14 +154,56 @@ abstract class ClassfileParser {
152154
def parse(file: AbstractFile, clazz: ClassSymbol, module: ModuleSymbol): Unit = {
153155
this.file = file
154156
pushBusy(clazz) {
155-
this.in = new AbstractFileReader(file)
156157
this.clazz = clazz
157158
this.staticModule = module
158159
this.isScala = false
159160

160-
parseHeader()
161-
this.pool = newConstantPool
162-
parseClass()
161+
import loaders.platform._
162+
classFileInfo(file, clazz) match {
163+
case Some(info) =>
164+
info match {
165+
case ScalaRawClass(className) =>
166+
isScalaRaw = true
167+
currentClass = TermName(className)
168+
case ScalaClass(className, pickle) =>
169+
val pickle1 = pickle
170+
isScala = true
171+
currentClass = TermName(className)
172+
if (pickle1.hasArray) {
173+
unpickler.unpickle(pickle1.array, pickle1.arrayOffset + pickle1.position(), clazz, staticModule, file.name)
174+
} else {
175+
val array = new Array[Byte](pickle1.remaining)
176+
pickle1.get(array)
177+
unpickler.unpickle(array, 0, clazz, staticModule, file.name)
178+
}
179+
case ClassBytes(data) =>
180+
val data1 = data.duplicate()
181+
val array = new Array[Byte](data1.remaining)
182+
data1.get(array)
183+
this.in = new AbstractFileReader(file, array)
184+
parseHeader()
185+
this.pool = newConstantPool
186+
parseClass()
187+
}
188+
case None =>
189+
this.in = new AbstractFileReader(file)
190+
parseHeader()
191+
this.pool = newConstantPool
192+
parseClass()
193+
if (!(isScala || isScalaRaw))
194+
loaders.platform.classFileInfoParsed(file, clazz, ClassBytes(ByteBuffer.wrap(in.buf)))
195+
}
196+
if (isScalaRaw && !isNothingOrNull) {
197+
unlinkRaw()
198+
}
199+
}
200+
}
201+
202+
private def unlinkRaw(): Unit = {
203+
val decls = clazz.enclosingPackage.info.decls
204+
for (c <- List(clazz, staticModule, staticModule.moduleClass)) {
205+
c.setInfo(NoType)
206+
decls.unlink(c)
163207
}
164208
}
165209

@@ -441,6 +485,15 @@ abstract class ClassfileParser {
441485
lookupClass(name)
442486
}
443487

488+
// TODO: remove after the next 2.13 milestone
489+
// A bug in the backend caused classes ending in `$` do get only a Scala marker attribute
490+
// instead of a ScalaSig and a Signature annotaiton. This went unnoticed because isScalaRaw
491+
// classes were parsed like Java classes. The below covers the cases in the std lib.
492+
private def isNothingOrNull = {
493+
val n = clazz.fullName.toString
494+
n == "scala.runtime.Nothing$" || n == "scala.runtime.Null$"
495+
}
496+
444497
def parseClass() {
445498
val jflags = readClassFlags()
446499
val sflags = jflags.toScalaFlags
@@ -890,8 +943,8 @@ abstract class ClassfileParser {
890943
case Some(san: AnnotationInfo) =>
891944
val bytes =
892945
san.assocs.find({ _._1 == nme.bytes }).get._2.asInstanceOf[ScalaSigBytes].bytes
893-
894946
unpickler.unpickle(bytes, 0, clazz, staticModule, in.file.name)
947+
loaders.platform.classFileInfoParsed(file, clazz, ScalaClass(this.currentClass.toString, ByteBuffer.wrap(bytes)))
895948
case None =>
896949
throw new RuntimeException("Scala class file does not contain Scala annotation")
897950
}
@@ -1216,6 +1269,7 @@ abstract class ClassfileParser {
12161269
in.skip(attrLen)
12171270
case tpnme.ScalaATTR =>
12181271
isScalaRaw = true
1272+
loaders.platform.classFileInfoParsed(file, clazz, ScalaRawClass(this.currentClass.toString))
12191273
case tpnme.InnerClassesATTR if !isScala =>
12201274
val entries = u2
12211275
for (i <- 0 until entries) {

‎src/compiler/scala/tools/nsc/typechecker/Analyzer.scala

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -112,11 +112,13 @@ trait Analyzer extends AnyRef
112112
try {
113113
val typer = newTyper(rootContext(unit))
114114
unit.body = typer.typed(unit.body)
115-
for (workItem <- unit.toCheck) workItem()
116-
if (settings.warnUnusedImport)
117-
warnUnusedImports(unit)
118-
if (settings.warnUnused.isSetByUser)
119-
new checkUnused(typer).apply(unit)
115+
if (!settings.Youtline.value) {
116+
for (workItem <- unit.toCheck) workItem()
117+
if (settings.warnUnusedImport)
118+
warnUnusedImports(unit)
119+
if (settings.warnUnused.isSetByUser)
120+
new checkUnused(typer).apply(unit)
121+
}
120122
}
121123
finally {
122124
unit.toCheck.clear()

‎src/compiler/scala/tools/nsc/typechecker/Typers.scala

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2057,8 +2057,9 @@ trait Typers extends Adaptations with Tags with TypersTracking with PatternTyper
20572057
}
20582058

20592059
// use typedValDef instead. this version is called after creating a new context for the ValDef
2060-
private def typedValDefImpl(vdef: ValDef) = {
2060+
private def typedValDefImpl(vdef: ValDef): ValDef = {
20612061
val sym = vdef.symbol.initialize
2062+
20622063
val typedMods = if (nme.isLocalName(sym.name) && sym.isPrivateThis && !vdef.mods.isPrivateLocal) {
20632064
// scala/bug#10009 This tree has been given a field symbol by `enterGetterSetter`, patch up the
20642065
// modifiers accordingly so that we can survive resetAttrs and retypechecking.
@@ -5845,7 +5846,7 @@ trait Typers extends Adaptations with Tags with TypersTracking with PatternTyper
58455846
final def transformedOrTyped(tree: Tree, mode: Mode, pt: Type): Tree = {
58465847
lookupTransformed(tree) match {
58475848
case Some(tree1) => tree1
5848-
case _ => typed(tree, mode, pt)
5849+
case _ => if (settings.Youtline.value) EmptyTree else typed(tree, mode, pt)
58495850
}
58505851
}
58515852
final def lookupTransformed(tree: Tree): Option[Tree] =

‎src/reflect/scala/reflect/internal/Symbols.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1978,7 +1978,7 @@ trait Symbols extends api.Symbols { self: SymbolTable =>
19781978
var alts0: List[Symbol] = alternatives
19791979
var alts1: List[Symbol] = Nil
19801980

1981-
while (alts0.nonEmpty) {
1981+
while (!alts0.isEmpty) {
19821982
if (cond(alts0.head))
19831983
alts1 ::= alts0.head
19841984
else

‎src/reflect/scala/reflect/internal/pickling/UnPickler.scala

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -249,6 +249,9 @@ abstract class UnPickler {
249249
else NoSymbol
250250
}
251251

252+
if (owner == definitions.ScalaPackageClass && name == tpnme.AnyRef)
253+
return definitions.AnyRefClass
254+
252255
// (1) Try name.
253256
localDummy orElse fromName(name) orElse {
254257
// (2) Try with expanded name. Can happen if references to private

‎test/junit/scala/tools/nsc/classpath/ClassPluginTest.scala

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
/*
2+
* Copyright (c) 2018 Lightbend. All rights reserved.
3+
*/
4+
package scala.tools.nsc.classpath
5+
6+
import java.nio.ByteBuffer
7+
8+
import org.junit.Assert.assertEquals
9+
import org.junit.Test
10+
import org.junit.runner.RunWith
11+
import org.junit.runners.JUnit4
12+
13+
import scala.reflect.io.VirtualDirectory
14+
import scala.tools.nsc.backend.{ClassfileInfo, ScalaClass}
15+
import scala.tools.nsc.io.AbstractFile
16+
import scala.tools.nsc.symtab.SymbolTableForUnitTesting
17+
import scala.tools.nsc.util.ClassPath
18+
import scala.tools.testing.BytecodeTesting
19+
import scala.tools.testing.BytecodeTesting.makeSourceFile
20+
21+
@RunWith(classOf[JUnit4])
22+
class ClassPluginTest extends BytecodeTesting {
23+
// We use this.compiler to generate Scala pickles...
24+
override def compilerArgs = "-Ystop-after:pickler"
25+
26+
// ... and this one to read them with a ClassPathPlugin
27+
object symbolTable extends SymbolTableForUnitTesting {
28+
val fakeClasses = Map(
29+
"fake.C" -> ScalaClass("fake.C", pickleOf("package fake; class C { def foo = 42 }"))
30+
)
31+
private val fakes = new VirtualDirectory("fakes", None)
32+
fakes.subdirectoryNamed("fake").fileNamed("C.class")
33+
34+
lazy val classpathPlugin = new platform.ClassPathPlugin {
35+
override def modifyClassPath(classPath: Seq[ClassPath]): Seq[ClassPath] = {
36+
// Add a classpath entry with the fake/C.class
37+
VirtualDirectoryClassPath(fakes) +: classPath
38+
}
39+
40+
override def info(file: AbstractFile, clazz: ClassSymbol): Option[ClassfileInfo] =
41+
fakeClasses.get(clazz.fullNameString)
42+
}
43+
this.platform.addClassPathPlugin(classpathPlugin)
44+
}
45+
46+
@Test def classPathPluginTest(): Unit = {
47+
import symbolTable._
48+
val CClass = rootMirror.getRequiredClass("fake.C")
49+
val C_tpe = CClass.info
50+
assertEquals("def foo: Int", definitions.fullyInitializeSymbol(C_tpe.decl(TermName("foo"))).defString)
51+
}
52+
53+
private def pickleOf(code: String): ByteBuffer = {
54+
import compiler._
55+
val run = newRun
56+
run.compileSources(makeSourceFile(code, "unitTestSource.scala") :: Nil)
57+
val pickle = run.symData.toList.head._2
58+
ByteBuffer.wrap(pickle.bytes, 0, pickle.writeIndex)
59+
}
60+
}

‎test/junit/scala/tools/nsc/symtab/SymbolTableForUnitTesting.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ class SymbolTableForUnitTesting extends SymbolTable {
3636

3737
def platformPhases: List[SubComponent] = Nil
3838

39-
private[nsc] lazy val classPath: ClassPath = new PathResolver(settings).result
39+
private[nsc] lazy val classPath: ClassPath = applyClassPathPlugins(new PathResolver(settings).result)
4040

4141
def isMaybeBoxed(sym: Symbol): Boolean = ???
4242
def needCompile(bin: AbstractFile, src: AbstractFile): Boolean = ???

0 commit comments

Comments
 (0)