diff --git a/src/compiler/scala/tools/nsc/PipelineMain.scala b/src/compiler/scala/tools/nsc/PipelineMain.scala new file mode 100644 index 000000000000..c1fd402e7d2b --- /dev/null +++ b/src/compiler/scala/tools/nsc/PipelineMain.scala @@ -0,0 +1,466 @@ +/* NSC -- new Scala compiler + * Copyright 2005-2018 LAMP/EPFL + * @author Martin Odersky + */ +package scala.tools.nsc + +import java.io.File +import java.nio.ByteBuffer +import java.nio.file.{Files, Path, Paths} +import java.util.Collections + +import javax.tools.{SimpleJavaFileObject, ToolProvider} + +import scala.collection.JavaConverters.asJavaIterableConverter +import scala.collection.{immutable, mutable, parallel} +import scala.concurrent.{Await, ExecutionContext, Future, Promise} +import scala.reflect.internal.pickling.PickleBuffer +import scala.reflect.internal.util.FakePos +import scala.reflect.io.{VirtualDirectory, VirtualFile} +import scala.tools.nsc.backend.{ClassfileInfo, JavaPlatform, ScalaClass, ScalaRawClass} +import scala.tools.nsc.classpath.{DirectoryClassPath, VirtualDirectoryClassPath, ZipArchiveFileLookup} +import scala.tools.nsc.io.AbstractFile +import scala.tools.nsc.reporters.{ConsoleReporter, Reporter} +import scala.tools.nsc.util.ClassPath +import scala.util.{Failure, Success} +import scala.concurrent.duration.Duration +import scala.tools.nsc.classpath.ZipAndJarClassPathFactory.ZipArchiveClassPath + +class PipelineMainClass(label: String, parallelism: Int, strategy: BuildStrategy) { + /** Forward errors to the (current) reporter. */ + protected def scalacError(msg: String): Unit = { + reporter.error(FakePos("scalac"), msg + "\n scalac -help gives more information") + } + + private var reporter: Reporter = _ + + implicit val executor = ExecutionContext.fromExecutor(new java.util.concurrent.ForkJoinPool(parallelism)) + val fileManager = ToolProvider.getSystemJavaCompiler.getStandardFileManager(null, null, null) + + private class PickleClassPath[G <: Global](data: mutable.AnyRefMap[G#Symbol, PickleBuffer]) { + val dir = new VirtualDirectory("fakes", None) + val classpath = VirtualDirectoryClassPath(dir) + val dirs = mutable.Map[G#Symbol, AbstractFile]() + val classInfo = mutable.Map[AbstractFile, ClassfileInfo]() + def packageDir(packSymbol: G#Symbol): AbstractFile = { + if (packSymbol.isEmptyPackageClass) dir + else if (dirs.contains(packSymbol)) dirs(packSymbol) + else if (packSymbol.owner.isRoot) { + val subDir = dir.subdirectoryNamed(packSymbol.encodedName) + dirs.put(packSymbol, subDir) + subDir + } else { + val base = packageDir(packSymbol.owner) + val subDir = base.subdirectoryNamed(packSymbol.encodedName) + dirs.put(packSymbol, subDir) + subDir + } + } + for ((symbol, pickle) <- data) { + val base = packageDir(symbol.owner) + if (symbol.isClass) { + val primary = base.fileNamed(symbol.encodedName + ".class") + classInfo(primary) = ScalaClass(symbol.fullNameString, () => ByteBuffer.wrap(pickle.bytes)) + if (symbol.companionModule.exists) { + val secondary = base.fileNamed(symbol.companionModule.encodedName + "$.class") + classInfo(secondary) = ScalaRawClass(symbol.companionModule.fullNameString) + } + } else if (symbol.isModule) { + if (symbol.companionClass.exists) { + val primary = base.fileNamed(symbol.encodedName + ".class") + classInfo(primary) = ScalaClass(symbol.fullNameString, () => ByteBuffer.wrap(pickle.bytes)) + val secondary = base.fileNamed(symbol.companionModule.encodedName + "$.class") + classInfo(secondary) = ScalaRawClass(symbol.companionModule.fullNameString) + } else { + val primary = base.fileNamed(symbol.encodedName + "$.class") + classInfo(primary) = ScalaClass(symbol.fullNameString, () => ByteBuffer.wrap(pickle.bytes)) + } + } + } + } + private val allPickleData = new java.util.concurrent.ConcurrentHashMap[Path, PickleClassPath[_]] + private val allParsedInfos = new java.util.concurrent.ConcurrentHashMap[AbstractFile, ClassfileInfo] + + def process(args: Array[String]): Boolean = { + println(s"parallelism = $parallelism, strategy = $strategy") + + reporter = new ConsoleReporter(new Settings(scalacError)) + + def commandFor(argFileArg: String): Task = { + val ss = new Settings(scalacError) + val command = new CompilerCommand(("@" + argFileArg) :: Nil, ss) + Task(argFileArg, command, command.files) + } + val projects: List[Task] = args.toList.map(commandFor) + val produces = mutable.HashMap[Path, Task]() + for (p <- projects) { + val outputDir = p.command.settings.outputDirs.getSingleOutput.get.file.toPath.toAbsolutePath.normalize() + produces(outputDir) = p + } + val dependsOn = mutable.HashMap[Task, List[Task]]() + for (p <- projects) { + val value: Seq[String] = ClassPath.expandPath(p.command.settings.classpath.value, expandStar = true) + dependsOn(p) = value.flatMap(s => produces.get(Paths.get(s).toAbsolutePath.normalize())).toList.filterNot(_ == p) + } + val dependedOn: Set[Task] = dependsOn.valuesIterator.flatten.toSet + val timer = new Timer + timer.start() + strategy match { + case OutlineTypeOnly => + val futures = projects.map { p => + val f1 = Future.sequence[Unit, List](dependsOn.getOrElse(p, Nil).map(_.outlineDone.future)) + p.shouldOutlineType = true + f1.map { _ => + p.outlineCompile() + p.javaCompile() + } + } + + val toWait: Future[List[Unit]] = Future.sequence(futures).flatMap(_ => Future.sequence(projects.flatMap(p => p.javaDone.future :: p.outlineDone.future :: Nil) )) + Await.result(toWait, Duration.Inf) + timer.stop() + + for (p <- projects) { + val dependencies = dependsOn(p) + def maxByOrZero[A](as: List[A])(f: A => Double): Double = if (as.isEmpty) 0d else as.map(f).max + val maxOutlineCriticalPathMs = maxByOrZero(dependencies)(_.outlineCriticalPathMs) + p.outlineCriticalPathMs = maxOutlineCriticalPathMs + p.outlineTimer.durationMs + p.regularCriticalPathMs = maxOutlineCriticalPathMs + maxByOrZero(p.groups)(_.timer.durationMs) + p.fullCriticalPathMs = maxByOrZero(dependencies)(_.fullCriticalPathMs) + p.groups.map(_.timer.durationMs).sum + } + + if (parallelism == 1) { + val criticalPath = projects.maxBy(_.regularCriticalPathMs) + println(f"Critical path: ${criticalPath.regularCriticalPathMs}%.0f ms. Wall Clock: ${timer.durationMs}%.0f ms") + } else + println(f" Wall Clock: ${timer.durationMs}%.0f ms") + case OutlineTypePipeline => + val futures = projects.map { p => + val f1 = Future.sequence[Unit, List](dependsOn.getOrElse(p, Nil).map(_.outlineDone.future)) + val shouldOutlineType = dependedOn(p) + p.shouldOutlineType = shouldOutlineType + f1.map { _ => + if (p.shouldOutlineType) { + p.outlineCompile() + } else { + p.fullCompile() + } + } + } + projects.map { + p => + if (p.shouldOutlineType) p.outlineDone.future.onComplete { _ => + p.fullCompile() + } + Future.sequence(p.groups.map(_.done.future)).map(_ => p.javaCompile()) + } + val toWait: Future[List[Unit]] = Future.sequence(futures).flatMap(_ => Future.sequence(projects.flatMap(p => p.javaDone.future :: p.groups.map(_.done.future) ))) + Await.result(toWait, Duration.Inf) + timer.stop() + + for (p <- projects) { + val dependencies = dependsOn(p) + def maxByOrZero[A](as: List[A])(f: A => Double): Double = if (as.isEmpty) 0d else as.map(f).max + val maxOutlineCriticalPathMs = maxByOrZero(dependencies)(_.outlineCriticalPathMs) + p.outlineCriticalPathMs = maxOutlineCriticalPathMs + p.outlineTimer.durationMs + p.regularCriticalPathMs = maxOutlineCriticalPathMs + maxByOrZero(p.groups)(_.timer.durationMs) + p.fullCriticalPathMs = maxByOrZero(dependencies)(_.fullCriticalPathMs) + p.groups.map(_.timer.durationMs).sum + } + + if (parallelism == 1) { + val criticalPath = projects.maxBy(_.regularCriticalPathMs) + println(f"Critical path: ${criticalPath.regularCriticalPathMs}%.0f ms. Wall Clock: ${timer.durationMs}%.0f ms") + } else + println(f" Wall Clock: ${timer.durationMs}%.0f ms") + case Pipeline => + val futures: immutable.Seq[Future[Unit]] = projects.map { p => + val f1 = Future.sequence[Unit, List](dependsOn.getOrElse(p, Nil).map(_.outlineDone.future)) + f1.map { _ => p.fullCompileExportPickles(); p.javaCompile() } + } + val toWait: Future[List[Unit]] = Future.sequence(futures).flatMap(_ => Future.sequence(projects.flatMap(p => p.javaDone.future :: p.groups.map(_.done.future) ))) + Await.result(toWait, Duration.Inf) + timer.stop() + + for (p <- projects) { + val dependencies = dependsOn(p) + def maxByOrZero[A](as: List[A])(f: A => Double): Double = if (as.isEmpty) 0d else as.map(f).max + val maxOutlineCriticalPathMs = maxByOrZero(dependencies)(_.outlineCriticalPathMs) + p.outlineCriticalPathMs = maxOutlineCriticalPathMs + p.outlineTimer.durationMs + p.regularCriticalPathMs = maxOutlineCriticalPathMs + maxByOrZero(p.groups)(_.timer.durationMs) + p.fullCriticalPathMs = maxByOrZero(dependencies)(_.fullCriticalPathMs) + p.groups.map(_.timer.durationMs).sum + } + + if (parallelism == 1) { + val criticalPath = projects.maxBy(_.regularCriticalPathMs) + println(f"Critical path: ${criticalPath.regularCriticalPathMs}%.0f ms. Wall Clock: ${timer.durationMs}%.0f ms") + } else + println(f" Wall Clock: ${timer.durationMs}%.0f ms") + case Traditional => + val futures = projects.map { p => + val f1 = Future.sequence[Unit, List](dependsOn.getOrElse(p, Nil).map(_.javaDone.future)) + val shouldOutlineType = dependedOn(p) + p.shouldOutlineType = shouldOutlineType + f1.flatMap { _ => + p.fullCompile() + Future.sequence(p.groups.map(_.done.future)).map(_ => p.javaCompile()) + } + } + val toWait: Future[List[Unit]] = Future.sequence(futures).flatMap(_ => Future.sequence(projects.flatMap(p => p.javaDone.future :: p.groups.map(_.done.future) ))) + Await.result(toWait, Duration.Inf) + timer.stop() + + for (p <- projects) { + val dependencies = dependsOn(p) + def maxByOrZero[A](as: List[A])(f: A => Double): Double = if (as.isEmpty) 0d else as.map(f).max + p.fullCriticalPathMs = maxByOrZero(dependencies)(_.fullCriticalPathMs) + p.groups.map(_.timer.durationMs).sum + } + if (parallelism == 1) { + val maxFullCriticalPath: Double = projects.map(_.fullCriticalPathMs).max + println(f"Critical path: $maxFullCriticalPath%.0f ms. Wall Clock: ${timer.durationMs}%.0f ms") + } else { + println(f"Wall Clock: ${timer.durationMs}%.0f ms") + } + } + + val trace = new java.lang.StringBuilder() + trace.append("""{"traceEvents": [""") + val sb = new mutable.StringBuilder(trace) + def durationEvent(name: String, cat: String, t: Timer): String = { + s"""{"name": "$name", "cat": "$cat", "ph": "X", "ts": ${(t.startMicros).toLong}, "dur": ${(t.durationMicros).toLong}, "pid": 0, "tid": ${t.thread.getId}}""" + } + def projectEvents(p: Task): List[String] = { + val events = List.newBuilder[String] + if (p.outlineTimer.durationMicros > 0d) { + events += durationEvent(p.label, "outline-type", p.outlineTimer) + } + for ((g, ix) <- p.groups.zipWithIndex) { + if (g.timer.durationMicros > 0d) + events += durationEvent(p.label, "compile-" + ix, g.timer) + } + events.result() + } + projects.iterator.flatMap(projectEvents).addString(sb, ",\n") + trace.append("]}") + Files.write(Paths.get(s"build-${label}.trace"), trace.toString.getBytes()) + true + } + + case class Group(files: List[String]) { + val timer = new Timer + val done = Promise[Unit]() + } + private case class Task(argsFile: String, command: CompilerCommand, files: List[String]) { + val label = argsFile.replaceAll("target/", "").replaceAll("""(.*)/(.*).args""", "$1:$2") + override def toString: String = argsFile + + command.settings.YcacheMacroClassLoader.value = "none" + + val groups: List[Group] = { + val isScalaLibrary = files.exists(_.endsWith("Predef.scala")) + if (strategy != OutlineTypePipeline || isScalaLibrary) { + Group(files) :: Nil + } else { + command.settings.classpath.value = command.settings.outputDirs.getSingleOutput.get.toString + File.pathSeparator + command.settings.classpath.value + val length = files.length + val groups = (length.toDouble / 128).toInt.max(1) + files.grouped((length.toDouble / groups).ceil.toInt.max(1)).toList.map(Group(_)) + } + } + command.settings.outputDirs.getSingleOutput.get.file.mkdirs() + + val isGrouped = groups.size > 1 + + val outlineTimer = new Timer() + + var shouldOutlineType = true + var outlineCriticalPathMs = 0d + var regularCriticalPathMs = 0d + var fullCriticalPathMs = 0d + val outlineDone = Promise[Unit]() + val javaDone = Promise[Unit]() + + lazy val compiler: Global = { + val result = newCompiler(command.settings) + val reporter = result.reporter + if (reporter.hasErrors) + reporter.flush() + else if (command.shouldStopWithInfo) + reporter.echo(command.getInfoMessage(compiler)) + result + } + + def outlineCompile(): Unit = { + outlineTimer.start() + command.settings.Youtline.value = true + command.settings.stopAfter.value = List("pickler") + command.settings.Ymacroexpand.value = command.settings.MacroExpand.None + val run1 = new compiler.Run() + run1 compile files + allPickleData.put(command.settings.outputDirs.getSingleOutput.get.file.toPath.toRealPath().normalize(), new PickleClassPath(run1.symData)) + outlineTimer.stop() + reporter.finish() + if (reporter.hasErrors) + outlineDone.complete(Failure(new RuntimeException("compile failed"))) + else + outlineDone.complete(Success(())) + } + + def fullCompile(): Unit = { + command.settings.Youtline.value = false + command.settings.stopAfter.value = Nil + command.settings.Ymacroexpand.value = command.settings.MacroExpand.Normal + + for (group <- groups) { + group.done.completeWith { + Future { + val compiler2 = newCompiler(command.settings) + val run2 = new compiler2.Run() + group.timer.start() + run2 compile group.files + compiler2.reporter.finish() + group.timer.stop() + if (compiler2.reporter.hasErrors) { + group.done.complete(Failure(new RuntimeException("Compile failed"))) + } else { + group.done.complete(Success(())) + } + } + } + } + } + + def fullCompileExportPickles(): Unit = { + assert(groups.size == 1) + val group = groups.head + outlineTimer.start() + val run2 = new compiler.Run() { + override def advancePhase(): Unit = { + if (compiler.phase == this.picklerPhase) { + allPickleData.put(command.settings.outputDirs.getSingleOutput.get.file.toPath.toRealPath().normalize(), new PickleClassPath(symData)) + outlineTimer.stop() + outlineDone.complete(Success(())) + group.timer.start() + } + super.advancePhase() + } + } + + run2 compile group.files + compiler.reporter.finish() + group.timer.stop() + if (compiler.reporter.hasErrors) { + group.done.complete(Failure(new RuntimeException("Compile failed"))) + } else { + group.done.complete(Success(())) + } + } + + def javaCompile(): Unit = { + val javaSources = files.filter(_.endsWith(".java")) + if (javaSources.nonEmpty) { + javaDone.completeWith(Future { + val opts = java.util.Arrays.asList("-d", command.settings.outdir.value, "-cp", command.settings.outdir.value + File.pathSeparator + command.settings.classpath.value) + val compileTask = ToolProvider.getSystemJavaCompiler.getTask(null, null, null, opts, null, fileManager.getJavaFileObjects(javaSources.toArray: _*)) + compileTask.setProcessors(Collections.emptyList()) + compileTask.call() + }) + } else { + javaDone.complete(Success(())) + } + } + } + + final class Timer() { + private var startNanos: Long = 0 + private var endNanos: Long = 0 + def start(): Unit = { + assert(startNanos == 0L) + startNanos = System.nanoTime + } + var thread: Thread = Thread.currentThread() + def stop(): Unit = { + thread = Thread.currentThread() + endNanos = System.nanoTime() + } + def startMs: Double = startNanos.toDouble / 1000 / 1000 + def durationMs: Double = (endNanos - startNanos).toDouble / 1000 / 1000 + def startMicros: Double = startNanos.toDouble / 1000d + def durationMicros: Double = (endNanos - startNanos).toDouble / 1000d + } + + protected def newCompiler(settings: Settings): Global = { + val g = Global(settings) + + val plugin: g.platform.ClassPathPlugin = new g.platform.ClassPathPlugin { + val replacements = mutable.Buffer[PickleClassPath[_]]() + override def modifyClassPath(classPath: Seq[ClassPath]): Seq[ClassPath] = { + classPath.flatMap { + case zcp: ZipArchiveFileLookup[_] => + val path = zcp.zipFile.toPath.toRealPath().normalize() + allPickleData.get(path) match { + case null => + zcp :: Nil + case pcp => + replacements += pcp + pcp.classpath :: zcp :: Nil // leaving the original classpath for Java compiled files for now + } + case dcp: DirectoryClassPath => + val path = dcp.dir.toPath.toRealPath().normalize() + allPickleData.get(path) match { + case null => + dcp :: Nil + case pcp => + replacements += pcp + pcp.classpath :: dcp :: Nil // leaving the original classpath for Java compiled files for now + } + case cp => cp :: Nil + } + } + + override def info(file: AbstractFile, clazz: g.ClassSymbol): Option[ClassfileInfo] = { + file match { + case vf: VirtualFile => + val iterator = replacements.iterator.flatMap(_.classInfo.get(vf)) + if (iterator.hasNext) + return Some(iterator.next()) + else None + case _ => None + } + allParsedInfos.get(file) match { + case null => None + case info => Some(info) + } + } + override def parsed(file: AbstractFile, clazz: g.ClassSymbol, info: ClassfileInfo): Unit = { + allParsedInfos.put(file, info) + } + } + g.platform.addClassPathPlugin(plugin) + g + } + +} + +sealed abstract class BuildStrategy +case object OutlineTypeOnly extends BuildStrategy +/** Outline type check to compute type signatures as pickles as an input to downstream compilation. */ +case object OutlineTypePipeline extends BuildStrategy +case object Pipeline extends BuildStrategy +/** Emit class files before triggering downstream compilation */ +case object Traditional extends BuildStrategy + +object PipelineMain { + def main(args: Array[String]): Unit = { + var i = 0 + //for (_ <- 1 to 10; n <- List(parallel.availableProcessors, 1); strat <- List(Pipeline, OutlineTypePipeline, Traditional)) { + for (_ <- 1 to 20; n <- List(parallel.availableProcessors); strat <- List(OutlineTypeOnly)) { + i += 1 + val main = new PipelineMainClass(i.toString, n, strat) + println(s"====== ITERATION $i=======") + val result = main.process(args) + if (!result) + System.exit(1) + } + System.exit(0) + } +} diff --git a/src/compiler/scala/tools/nsc/backend/JavaPlatform.scala b/src/compiler/scala/tools/nsc/backend/JavaPlatform.scala index ff11f434710b..4d1bd6d8ece4 100644 --- a/src/compiler/scala/tools/nsc/backend/JavaPlatform.scala +++ b/src/compiler/scala/tools/nsc/backend/JavaPlatform.scala @@ -27,7 +27,7 @@ trait JavaPlatform extends Platform { private[nsc] var currentClassPath: Option[ClassPath] = None private[nsc] def classPath: ClassPath = { - if (currentClassPath.isEmpty) currentClassPath = Some(new PathResolver(settings).result) + if (currentClassPath.isEmpty) currentClassPath = Some(applyClassPathPlugins(new PathResolver(settings).result)) currentClassPath.get } diff --git a/src/compiler/scala/tools/nsc/backend/Platform.scala b/src/compiler/scala/tools/nsc/backend/Platform.scala index a69e79d4c4f1..e0d4d2c59d09 100644 --- a/src/compiler/scala/tools/nsc/backend/Platform.scala +++ b/src/compiler/scala/tools/nsc/backend/Platform.scala @@ -13,7 +13,10 @@ package scala.tools.nsc package backend +import java.nio.ByteBuffer + import io.AbstractFile +import scala.tools.nsc.classpath.AggregateClassPath import scala.tools.nsc.util.ClassPath /** The platform dependent pieces of Global. @@ -44,5 +47,57 @@ trait Platform { * a re-compile is triggered. On .NET by contrast classfiles always take precedence. */ def needCompile(bin: AbstractFile, src: AbstractFile): Boolean + + /** + * A class path plugin can modify the classpath before it is used by the compiler, and can + * customize the way that the compiler reads the contents of class files. + * + * Applications could include: + * + * - Caching the ScalaSignature annotation contents, to avoid the cost of decompressing + * and parsing the classfile, akin to the OpenJDK's .sig format for stripped class files. + * - Starting a downstream compilation job immediately after the upstream job has completed + * the pickler phase ("Build Pipelineing") + */ + abstract class ClassPathPlugin { + def info(file: AbstractFile, clazz: ClassSymbol): Option[ClassfileInfo] + def parsed(file: AbstractFile, clazz: ClassSymbol, info: ClassfileInfo): Unit = () + def modifyClassPath(classPath: Seq[ClassPath]): Seq[ClassPath] = classPath + } + + /** A list of registered classpath plugins */ + private var classPathPlugins: List[ClassPathPlugin] = Nil + + protected final def applyClassPathPlugins(original: ClassPath): ClassPath = { + val entries = original match { + case AggregateClassPath(entries) => entries + case single => single :: Nil + } + val entries1 = classPathPlugins.foldLeft(entries) { + (entries, plugin) => plugin.modifyClassPath(entries) + } + AggregateClassPath(entries1) + } + + + /** Registers a new classpath plugin */ + final def addClassPathPlugin(plugin: ClassPathPlugin): Unit = { + if (!classPathPlugins.contains(plugin)) + classPathPlugins = plugin :: classPathPlugins + } + final def classFileInfo(file: AbstractFile, clazz: ClassSymbol): Option[ClassfileInfo] = if (classPathPlugins eq Nil) None else { + classPathPlugins.foldLeft(Option.empty[ClassfileInfo]) { + case (Some(info), _) => Some(info) + case (None, plugin) => plugin.info(file, clazz) + } + } + final def classFileInfoParsed(file: AbstractFile, clazz: ClassSymbol, info: ClassfileInfo): Unit = if (classPathPlugins eq Nil) None else { + classPathPlugins.foreach(_.parsed(file, clazz, info)) + } } +sealed abstract class ClassfileInfo {} +final case class ClassBytes(data: () => ByteBuffer) extends ClassfileInfo +final case class ScalaRawClass(className: String) extends ClassfileInfo +final case class ScalaClass(className: String, pickle: () => ByteBuffer) extends ClassfileInfo + diff --git a/src/compiler/scala/tools/nsc/classpath/VirtualDirectoryClassPath.scala b/src/compiler/scala/tools/nsc/classpath/VirtualDirectoryClassPath.scala index 5b157e9b386e..04ddc61b2107 100644 --- a/src/compiler/scala/tools/nsc/classpath/VirtualDirectoryClassPath.scala +++ b/src/compiler/scala/tools/nsc/classpath/VirtualDirectoryClassPath.scala @@ -35,7 +35,7 @@ case class VirtualDirectoryClassPath(dir: VirtualDirectory) extends ClassPath wi def isPackage(f: AbstractFile): Boolean = f.isPackage // mimic the behavior of the old nsc.util.DirectoryClassPath - def asURLs: Seq[URL] = Seq(new URL(dir.name)) + def asURLs: Seq[URL] = Seq(new URL("file://_VIRTUAL_/" + dir.name)) def asClassPathStrings: Seq[String] = Seq(dir.path) override def findClass(className: String): Option[ClassRepresentation] = findClassFile(className) map ClassFileEntryImpl diff --git a/src/compiler/scala/tools/nsc/settings/AbsSettings.scala b/src/compiler/scala/tools/nsc/settings/AbsSettings.scala index 64eeb8717a9b..500f4bea606d 100644 --- a/src/compiler/scala/tools/nsc/settings/AbsSettings.scala +++ b/src/compiler/scala/tools/nsc/settings/AbsSettings.scala @@ -25,7 +25,7 @@ trait AbsSettings extends scala.reflect.internal.settings.AbsSettings { protected def allSettings: scala.collection.Set[Setting] // settings minus internal usage settings - def visibleSettings = allSettings filterNot (_.isInternalOnly) + def visibleSettings = allSettings.iterator filterNot (_.isInternalOnly) // only settings which differ from default def userSetSettings = visibleSettings filterNot (_.isDefault) diff --git a/src/compiler/scala/tools/nsc/settings/ScalaSettings.scala b/src/compiler/scala/tools/nsc/settings/ScalaSettings.scala index 655c3528d18f..9dbb2699a115 100644 --- a/src/compiler/scala/tools/nsc/settings/ScalaSettings.scala +++ b/src/compiler/scala/tools/nsc/settings/ScalaSettings.scala @@ -243,6 +243,7 @@ trait ScalaSettings extends AbsScalaSettings val YcacheMacroClassLoader = CachePolicy.setting("macro", "macros") val YpartialUnification = BooleanSetting ("-Ypartial-unification", "Enable partial unification in type constructor inference") val Yvirtpatmat = BooleanSetting ("-Yvirtpatmat", "Enable pattern matcher virtualization") + val Youtline = BooleanSetting ("-Youtline", "Don't compile method bodies. Use together with `-Ystop-afer:pickler to generate the pickled signatures for all source files.") val exposeEmptyPackage = BooleanSetting ("-Yexpose-empty-package", "Internal only: expose the empty package.").internalOnly() val Ydelambdafy = ChoiceSetting ("-Ydelambdafy", "strategy", "Strategy used for translating lambdas into JVM code.", List("inline", "method"), "method") diff --git a/src/compiler/scala/tools/nsc/symtab/SymbolLoaders.scala b/src/compiler/scala/tools/nsc/symtab/SymbolLoaders.scala index 6444823efced..a907a33c4211 100644 --- a/src/compiler/scala/tools/nsc/symtab/SymbolLoaders.scala +++ b/src/compiler/scala/tools/nsc/symtab/SymbolLoaders.scala @@ -277,6 +277,8 @@ abstract class SymbolLoaders { val classPathEntries = classPath.list(packageName) + if (root.name.string_==("immutable")) + getClass if (!root.isRoot) for (entry <- classPathEntries.classesAndSources) initializeFromClassPath(root, entry) if (!root.isEmptyPackageClass) { diff --git a/src/compiler/scala/tools/nsc/symtab/classfile/AbstractFileReader.scala b/src/compiler/scala/tools/nsc/symtab/classfile/AbstractFileReader.scala index a8d673663e8d..19be00dd686a 100644 --- a/src/compiler/scala/tools/nsc/symtab/classfile/AbstractFileReader.scala +++ b/src/compiler/scala/tools/nsc/symtab/classfile/AbstractFileReader.scala @@ -25,11 +25,8 @@ import scala.tools.nsc.io.AbstractFile * @author Philippe Altherr * @version 1.0, 23/03/2004 */ -class AbstractFileReader(val file: AbstractFile) { - - /** the buffer containing the file - */ - val buf: Array[Byte] = file.toByteArray +class AbstractFileReader(val file: AbstractFile, val buf: Array[Byte]) { + def this(file: AbstractFile) = this(file, file.toByteArray) /** the current input pointer */ diff --git a/src/compiler/scala/tools/nsc/symtab/classfile/ClassfileParser.scala b/src/compiler/scala/tools/nsc/symtab/classfile/ClassfileParser.scala index 81f8dfe44543..407073b966f8 100644 --- a/src/compiler/scala/tools/nsc/symtab/classfile/ClassfileParser.scala +++ b/src/compiler/scala/tools/nsc/symtab/classfile/ClassfileParser.scala @@ -15,16 +15,18 @@ package tools.nsc package symtab package classfile -import java.io.{ByteArrayInputStream, DataInputStream, File, IOException} +import java.io._ import java.lang.Integer.toHexString +import java.nio.ByteBuffer import scala.collection.{immutable, mutable} import scala.collection.mutable.{ArrayBuffer, ListBuffer} import scala.annotation.switch import scala.reflect.internal.JavaAccFlags import scala.reflect.internal.pickling.{ByteCodecs, PickleBuffer} -import scala.reflect.io.NoAbstractFile +import scala.reflect.io.{NoAbstractFile, VirtualFile} import scala.reflect.internal.util.Collections._ +import scala.tools.nsc.backend.{ClassBytes, ScalaClass, ScalaRawClass} import scala.tools.nsc.util.ClassPath import scala.tools.nsc.io.AbstractFile import scala.util.control.NonFatal @@ -152,14 +154,45 @@ abstract class ClassfileParser { def parse(file: AbstractFile, clazz: ClassSymbol, module: ModuleSymbol): Unit = { this.file = file pushBusy(clazz) { - this.in = new AbstractFileReader(file) this.clazz = clazz this.staticModule = module this.isScala = false - parseHeader() - this.pool = newConstantPool - parseClass() + import loaders.platform._ + classFileInfo(file, clazz) match { + case Some(info) => + info match { + case ScalaRawClass(className) => + isScalaRaw = true + currentClass = TermName(className) + case ScalaClass(className, pickle) => + val pickle1 = pickle() + isScala = true + currentClass = TermName(className) + if (pickle1.hasArray) { + unpickler.unpickle(pickle1.array, pickle1.arrayOffset + pickle1.position(), clazz, staticModule, file.name) + } else { + val array = new Array[Byte](pickle1.remaining) + pickle1.get(array) + unpickler.unpickle(array, 0, clazz, staticModule, file.name) + } + case ClassBytes(data) => + val data1 = data() + val array = new Array[Byte](data1.remaining) + data1.get(array) + this.in = new AbstractFileReader(file, array) + parseHeader() + this.pool = newConstantPool + parseClass() + } + case None => + this.in = new AbstractFileReader(file) + parseHeader() + this.pool = newConstantPool + parseClass() + if (!(isScala || isScalaRaw)) + loaders.platform.classFileInfoParsed(file, clazz, ClassBytes(() => ByteBuffer.wrap(in.buf))) + } } } @@ -441,6 +474,15 @@ abstract class ClassfileParser { lookupClass(name) } + // TODO: remove after the next 2.13 milestone + // A bug in the backend caused classes ending in `$` do get only a Scala marker attribute + // instead of a ScalaSig and a Signature annotaiton. This went unnoticed because isScalaRaw + // classes were parsed like Java classes. The below covers the cases in the std lib. + private def isNothingOrNull = { + val n = clazz.fullName.toString + n == "scala.runtime.Nothing$" || n == "scala.runtime.Null$" + } + def parseClass() { val jflags = readClassFlags() val sflags = jflags.toScalaFlags @@ -890,8 +932,8 @@ abstract class ClassfileParser { case Some(san: AnnotationInfo) => val bytes = san.assocs.find({ _._1 == nme.bytes }).get._2.asInstanceOf[ScalaSigBytes].bytes - unpickler.unpickle(bytes, 0, clazz, staticModule, in.file.name) + loaders.platform.classFileInfoParsed(file, clazz, ScalaClass(this.currentClass.toString, () => ByteBuffer.wrap(bytes))) case None => throw new RuntimeException("Scala class file does not contain Scala annotation") } @@ -1216,6 +1258,7 @@ abstract class ClassfileParser { in.skip(attrLen) case tpnme.ScalaATTR => isScalaRaw = true + loaders.platform.classFileInfoParsed(file, clazz, ScalaRawClass(this.currentClass.toString)) case tpnme.InnerClassesATTR if !isScala => val entries = u2 for (i <- 0 until entries) { diff --git a/src/compiler/scala/tools/nsc/typechecker/Analyzer.scala b/src/compiler/scala/tools/nsc/typechecker/Analyzer.scala index b068e43d1ad4..bc5ffd0ccd7c 100644 --- a/src/compiler/scala/tools/nsc/typechecker/Analyzer.scala +++ b/src/compiler/scala/tools/nsc/typechecker/Analyzer.scala @@ -112,11 +112,13 @@ trait Analyzer extends AnyRef try { val typer = newTyper(rootContext(unit)) unit.body = typer.typed(unit.body) - for (workItem <- unit.toCheck) workItem() - if (settings.warnUnusedImport) - warnUnusedImports(unit) - if (settings.warnUnused.isSetByUser) - new checkUnused(typer).apply(unit) + if (!settings.Youtline.value) { + for (workItem <- unit.toCheck) workItem() + if (settings.warnUnusedImport) + warnUnusedImports(unit) + if (settings.warnUnused.isSetByUser) + new checkUnused(typer).apply(unit) + } } finally { unit.toCheck.clear() diff --git a/src/compiler/scala/tools/nsc/typechecker/Typers.scala b/src/compiler/scala/tools/nsc/typechecker/Typers.scala index acac49cff07d..027128514711 100644 --- a/src/compiler/scala/tools/nsc/typechecker/Typers.scala +++ b/src/compiler/scala/tools/nsc/typechecker/Typers.scala @@ -2057,8 +2057,9 @@ trait Typers extends Adaptations with Tags with TypersTracking with PatternTyper } // use typedValDef instead. this version is called after creating a new context for the ValDef - private def typedValDefImpl(vdef: ValDef) = { + private def typedValDefImpl(vdef: ValDef): ValDef = { val sym = vdef.symbol.initialize + val typedMods = if (nme.isLocalName(sym.name) && sym.isPrivateThis && !vdef.mods.isPrivateLocal) { // scala/bug#10009 This tree has been given a field symbol by `enterGetterSetter`, patch up the // modifiers accordingly so that we can survive resetAttrs and retypechecking. @@ -5845,7 +5846,7 @@ trait Typers extends Adaptations with Tags with TypersTracking with PatternTyper final def transformedOrTyped(tree: Tree, mode: Mode, pt: Type): Tree = { lookupTransformed(tree) match { case Some(tree1) => tree1 - case _ => typed(tree, mode, pt) + case _ => if (settings.Youtline.value) EmptyTree else typed(tree, mode, pt) } } final def lookupTransformed(tree: Tree): Option[Tree] = diff --git a/src/reflect/scala/reflect/internal/Symbols.scala b/src/reflect/scala/reflect/internal/Symbols.scala index 3a25d830a20f..b8c54bfe2934 100644 --- a/src/reflect/scala/reflect/internal/Symbols.scala +++ b/src/reflect/scala/reflect/internal/Symbols.scala @@ -1978,7 +1978,7 @@ trait Symbols extends api.Symbols { self: SymbolTable => var alts0: List[Symbol] = alternatives var alts1: List[Symbol] = Nil - while (alts0.nonEmpty) { + while (!alts0.isEmpty) { if (cond(alts0.head)) alts1 ::= alts0.head else diff --git a/src/reflect/scala/reflect/internal/pickling/UnPickler.scala b/src/reflect/scala/reflect/internal/pickling/UnPickler.scala index 7b82aa3e9f24..c1fc858cef11 100644 --- a/src/reflect/scala/reflect/internal/pickling/UnPickler.scala +++ b/src/reflect/scala/reflect/internal/pickling/UnPickler.scala @@ -249,6 +249,9 @@ abstract class UnPickler { else NoSymbol } + if (owner == definitions.ScalaPackageClass && name == tpnme.AnyRef) + return definitions.AnyRefClass + // (1) Try name. localDummy orElse fromName(name) orElse { // (2) Try with expanded name. Can happen if references to private diff --git a/src/repl/scala/tools/nsc/interpreter/ILoop.scala b/src/repl/scala/tools/nsc/interpreter/ILoop.scala index a32e2aa02ee7..7e4ac310c4d4 100644 --- a/src/repl/scala/tools/nsc/interpreter/ILoop.scala +++ b/src/repl/scala/tools/nsc/interpreter/ILoop.scala @@ -363,7 +363,8 @@ class ILoop(in0: Option[BufferedReader], protected val out: JPrintWriter) extend } private def changeSettings(line: String): Result = { - def showSettings() = for (s <- settings.userSetSettings.toSeq.sorted) echo(s.toString) + val settings1 = settings + def showSettings() = for (s <- settings1.userSetSettings.toSeq.sorted) echo(s.toString) if (line.isEmpty) showSettings() else { updateSettings(line) ; () } } private def updateSettings(line: String) = { diff --git a/test/junit/scala/tools/nsc/classpath/ClassPluginTest.scala b/test/junit/scala/tools/nsc/classpath/ClassPluginTest.scala new file mode 100644 index 000000000000..514294e8e8c1 --- /dev/null +++ b/test/junit/scala/tools/nsc/classpath/ClassPluginTest.scala @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2018 Lightbend. All rights reserved. + */ +package scala.tools.nsc.classpath + +import java.nio.ByteBuffer + +import org.junit.Assert.assertEquals +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 + +import scala.reflect.io.VirtualDirectory +import scala.tools.nsc.backend.{ClassfileInfo, ScalaClass} +import scala.tools.nsc.io.AbstractFile +import scala.tools.nsc.symtab.SymbolTableForUnitTesting +import scala.tools.nsc.util.ClassPath +import scala.tools.testing.BytecodeTesting +import scala.tools.testing.BytecodeTesting.makeSourceFile + +@RunWith(classOf[JUnit4]) +class ClassPluginTest extends BytecodeTesting { + // We use this.compiler to generate Scala pickles... + override def compilerArgs = "-Ystop-after:pickler" + + // ... and this one to read them with a ClassPathPlugin + object symbolTable extends SymbolTableForUnitTesting { + val fakeClasses = Map( + "fake.C" -> ScalaClass("fake.C", pickleOf("package fake; class C { def foo = 42 }")) + ) + private val fakes = new VirtualDirectory("fakes", None) + fakes.subdirectoryNamed("fake").fileNamed("C.class") + + lazy val classpathPlugin = new platform.ClassPathPlugin { + override def modifyClassPath(classPath: Seq[ClassPath]): Seq[ClassPath] = { + // Add a classpath entry with the fake/C.class + VirtualDirectoryClassPath(fakes) +: classPath + } + + override def info(file: AbstractFile, clazz: ClassSymbol): Option[ClassfileInfo] = + fakeClasses.get(clazz.fullNameString) + } + this.platform.addClassPathPlugin(classpathPlugin) + } + + @Test def classPathPluginTest(): Unit = { + import symbolTable._ + val CClass = rootMirror.getRequiredClass("fake.C") + val C_tpe = CClass.info + assertEquals("def foo: Int", definitions.fullyInitializeSymbol(C_tpe.decl(TermName("foo"))).defString) + } + + private def pickleOf(code: String): ByteBuffer = { + import compiler._ + val run = newRun + run.compileSources(makeSourceFile(code, "unitTestSource.scala") :: Nil) + val pickle = run.symData.toList.head._2 + ByteBuffer.wrap(pickle.bytes, 0, pickle.writeIndex) + } +} diff --git a/test/junit/scala/tools/nsc/symtab/SymbolTableForUnitTesting.scala b/test/junit/scala/tools/nsc/symtab/SymbolTableForUnitTesting.scala index cbd5634f292f..bbbc0ec4d882 100644 --- a/test/junit/scala/tools/nsc/symtab/SymbolTableForUnitTesting.scala +++ b/test/junit/scala/tools/nsc/symtab/SymbolTableForUnitTesting.scala @@ -36,7 +36,7 @@ class SymbolTableForUnitTesting extends SymbolTable { def platformPhases: List[SubComponent] = Nil - private[nsc] lazy val classPath: ClassPath = new PathResolver(settings).result + private[nsc] lazy val classPath: ClassPath = applyClassPathPlugins(new PathResolver(settings).result) def isMaybeBoxed(sym: Symbol): Boolean = ??? def needCompile(bin: AbstractFile, src: AbstractFile): Boolean = ???