diff --git a/project/ScriptCommands.scala b/project/ScriptCommands.scala index a5564242ebf3..4a4003066a70 100644 --- a/project/ScriptCommands.scala +++ b/project/ScriptCommands.scala @@ -109,11 +109,11 @@ object ScriptCommands { Project.setProject(session, newStructure, state) } - private[this] val enableOptimizer = Seq( + val enableOptimizer = Seq( scalacOptions in Compile in ThisBuild ++= Seq("-opt:l:inline", "-opt-inline-from:scala/**") ) - private[this] val noDocs = Seq( + val noDocs = Seq( publishArtifact in (Compile, packageDoc) in ThisBuild := false ) diff --git a/project/headershim.scala b/project/headershim.scala new file mode 100644 index 000000000000..0afa9d1be05e --- /dev/null +++ b/project/headershim.scala @@ -0,0 +1,34 @@ +package de.heikoseeberger.sbtheader + +import sbt._ +import sbt.Keys._ +import sbt.plugins.JvmPlugin + +object HeaderPlugin extends AutoPlugin { + + final object autoImport { + class License + object HeaderLicense { + case class Custom(s: String) extends License + } + val headerLicense: SettingKey[Option[License]] = settingKey[Option[License]]("header License") + + val headerSources = taskKey[scala.collection.Seq[File]]("Sources which need headers checked or created.") + + val headerResources = taskKey[scala.collection.Seq[File]]("Resources which need headers checked or created.") + + def headerSettings(configurations: Configuration*): Seq[Setting[_]] = + configurations.foldLeft(List.empty[Setting[_]]) { _ ++ inConfig(_)(toBeScopedSettings) } + } + + import autoImport._ + + override def trigger = allRequirements + + override def requires = JvmPlugin + + override def projectSettings = headerSettings(Compile, Test) + + private def toBeScopedSettings = Vector(headerSources := Nil, headerResources := Nil) + +} diff --git a/project/plugins.sbt b/project/plugins.sbt index 7a95a915a763..351c52084c6a 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -29,5 +29,3 @@ libraryDependencies ++= Seq( concurrentRestrictions in Global := Seq( Tags.limitAll(1) // workaround for https://github.com/sbt/sbt/issues/2970 ) - -addSbtPlugin("de.heikoseeberger" % "sbt-header" % "5.0.0") diff --git a/src/compiler/scala/reflect/macros/runtime/MacroRuntimes.scala b/src/compiler/scala/reflect/macros/runtime/MacroRuntimes.scala index 73520dffb925..557385744ac2 100644 --- a/src/compiler/scala/reflect/macros/runtime/MacroRuntimes.scala +++ b/src/compiler/scala/reflect/macros/runtime/MacroRuntimes.scala @@ -54,19 +54,8 @@ trait MacroRuntimes extends JavaReflectionRuntimes { /** Macro classloader that is used to resolve and run macro implementations. * Loads classes from from -cp (aka the library classpath). * Is also capable of detecting REPL and reusing its classloader. - * - * When -Xmacro-jit is enabled, we sometimes fallback to on-the-fly compilation of macro implementations, - * which compiles implementations into a virtual directory (very much like REPL does) and then conjures - * a classloader mapped to that virtual directory. */ - private lazy val defaultMacroClassloaderCache = { - def attemptClose(loader: ClassLoader): Unit = loader match { - case u: URLClassLoader => debuglog("Closing macro runtime classloader"); u.close() - case afcl: AbstractFileClassLoader => attemptClose(afcl.getParent) - case _ => ??? - } - perRunCaches.newGeneric(findMacroClassLoader, attemptClose _) - } + private lazy val defaultMacroClassloaderCache: () => ClassLoader = perRunCaches.newGeneric(findMacroClassLoader()) def defaultMacroClassloader: ClassLoader = defaultMacroClassloaderCache() /** Abstracts away resolution of macro runtimes. diff --git a/src/compiler/scala/tools/nsc/CloseableRegistry.scala b/src/compiler/scala/tools/nsc/CloseableRegistry.scala new file mode 100644 index 000000000000..9812a2136263 --- /dev/null +++ b/src/compiler/scala/tools/nsc/CloseableRegistry.scala @@ -0,0 +1,34 @@ +/* + * Scala (https://www.scala-lang.org) + * + * Copyright EPFL and Lightbend, Inc. + * + * Licensed under Apache License 2.0 + * (http://www.apache.org/licenses/LICENSE-2.0). + * + * See the NOTICE file distributed with this work for + * additional information regarding copyright ownership. + */ + +package scala.tools.nsc + +import scala.util.control.NonFatal + +/** Registry for resources to close when `Global` is closed */ +final class CloseableRegistry { + private[this] var closeables: List[java.io.Closeable] = Nil + final def registerClosable(c: java.io.Closeable): Unit = { + closeables ::= c + } + + def close(): Unit = { + for (c <- closeables) { + try { + c.close() + } catch { + case NonFatal(_) => + } + } + closeables = Nil + } +} diff --git a/src/compiler/scala/tools/nsc/CompilationUnits.scala b/src/compiler/scala/tools/nsc/CompilationUnits.scala index 159021bdacaf..46386beb58e7 100644 --- a/src/compiler/scala/tools/nsc/CompilationUnits.scala +++ b/src/compiler/scala/tools/nsc/CompilationUnits.scala @@ -21,7 +21,7 @@ trait CompilationUnits { global: Global => /** An object representing a missing compilation unit. */ object NoCompilationUnit extends CompilationUnit(NoSourceFile) { - override lazy val isJava = false + override val isJava = false override def exists = false override def toString() = "NoCompilationUnit" } @@ -153,7 +153,7 @@ trait CompilationUnits { global: Global => final def comment(pos: Position, msg: String): Unit = {} /** Is this about a .java source file? */ - lazy val isJava = source.file.name.endsWith(".java") + val isJava = source.file.name.endsWith(".java") override def toString() = source.toString() } diff --git a/src/compiler/scala/tools/nsc/GenericRunnerSettings.scala b/src/compiler/scala/tools/nsc/GenericRunnerSettings.scala index cb26b4d9d666..fcc829b2e64d 100644 --- a/src/compiler/scala/tools/nsc/GenericRunnerSettings.scala +++ b/src/compiler/scala/tools/nsc/GenericRunnerSettings.scala @@ -16,7 +16,14 @@ import java.net.URL import scala.tools.util.PathResolver class GenericRunnerSettings(error: String => Unit) extends Settings(error) { - lazy val classpathURLs: Seq[URL] = new PathResolver(this).resultAsURLs + lazy val classpathURLs: Seq[URL] = { + val registry = new CloseableRegistry + try { + new PathResolver(this, registry).resultAsURLs + } finally { + registry.close() + } + } val howtorun = ChoiceSetting( diff --git a/src/compiler/scala/tools/nsc/Global.scala b/src/compiler/scala/tools/nsc/Global.scala index a115eac0927b..efcfd8d2a358 100644 --- a/src/compiler/scala/tools/nsc/Global.scala +++ b/src/compiler/scala/tools/nsc/Global.scala @@ -40,9 +40,12 @@ import scala.language.postfixOps import scala.tools.nsc.ast.{TreeGen => AstTreeGen} import scala.tools.nsc.classpath._ import scala.tools.nsc.profile.Profiler +import scala.util.control.NonFatal +import java.io.Closeable class Global(var currentSettings: Settings, reporter0: Reporter) extends SymbolTable + with Closeable with CompilationUnits with Plugins with PhaseAssembly @@ -402,12 +405,16 @@ class Global(var currentSettings: Settings, reporter0: Reporter) def apply(unit: CompilationUnit): Unit + // run only the phases needed + protected def shouldSkipThisPhaseForJava: Boolean = { + this.id > (if (createJavadoc) currentRun.typerPhase.id + else currentRun.namerPhase.id) + } + /** Is current phase cancelled on this unit? */ def cancelled(unit: CompilationUnit) = { - // run the typer only if in `createJavadoc` mode - val maxJavaPhase = if (createJavadoc) currentRun.typerPhase.id else currentRun.namerPhase.id if (Thread.interrupted()) reporter.cancelled = true - reporter.cancelled || unit.isJava && this.id > maxJavaPhase + reporter.cancelled || unit.isJava && shouldSkipThisPhaseForJava } private def beforeUnit(unit: CompilationUnit): Unit = { @@ -446,8 +453,10 @@ class Global(var currentSettings: Settings, reporter0: Reporter) currentRun.informUnitStarting(this, unit) val unit0 = currentUnit currentRun.currentUnit = unit + currentRun.profiler.beforeUnit(phase, unit.source.file) try apply(unit) finally { + currentRun.profiler.afterUnit(phase, unit.source.file) currentRun.currentUnit = unit0 currentRun.advanceUnit() } @@ -817,7 +826,7 @@ class Global(var currentSettings: Settings, reporter0: Reporter) /** Extend classpath of `platform` and rescan updated packages. */ def extendCompilerClassPath(urls: URL*): Unit = { - val urlClasspaths = urls.map(u => ClassPathFactory.newClassPath(AbstractFile.getURL(u), settings)) + val urlClasspaths = urls.map(u => ClassPathFactory.newClassPath(AbstractFile.getURL(u), settings, closeableRegistry)) val newClassPath = AggregateClassPath.createAggregate(platform.classPath +: urlClasspaths : _*) platform.currentClassPath = Some(newClassPath) invalidateClassPathEntries(urls.map(_.getPath): _*) @@ -879,7 +888,7 @@ class Global(var currentSettings: Settings, reporter0: Reporter) } entries(classPath) find matchesCanonical match { case Some(oldEntry) => - Some(oldEntry -> ClassPathFactory.newClassPath(dir, settings)) + Some(oldEntry -> ClassPathFactory.newClassPath(dir, settings, closeableRegistry)) case None => error(s"Error adding entry to classpath. During invalidation, no entry named $path in classpath $classPath") None @@ -1110,6 +1119,9 @@ class Global(var currentSettings: Settings, reporter0: Reporter) def newJavaUnitParser(unit: CompilationUnit): JavaUnitParser = new JavaUnitParser(unit) + override protected[scala] def currentRunProfilerBeforeCompletion(root: Symbol, associatedFile: AbstractFile): Unit = currentRun.profiler.beforeCompletion(root, associatedFile) + override protected[scala] def currentRunProfilerAfterCompletion(root: Symbol, associatedFile: AbstractFile): Unit = currentRun.profiler.afterCompletion(root, associatedFile) + /** A Run is a single execution of the compiler on a set of units. */ class Run extends RunContextApi with RunReporting with RunParsing { @@ -1438,7 +1450,20 @@ class Global(var currentSettings: Settings, reporter0: Reporter) private def printArgs(sources: List[SourceFile]): Unit = { if (settings.printArgs.isSetByUser) { - val argsFile = (settings.recreateArgs ::: sources.map(_.file.absolute.toString())).mkString("", "\n", "\n") + val singleOuputDir: List[String] = if (settings.d.value == settings.d.default) { + settings.outputDirs.getSingleOutput match { + case Some(file) => + val jfile = file.file + if (jfile != null && !java.nio.file.Files.isSameFile(jfile.toPath, java.nio.file.Paths.get(settings.d.value))) { + // A build tool must have used `settings.outDirs.setSingleOutput`, bypassing `-d`. + // Render that to the equivalent -d arguments. + "-d" :: jfile.toString :: Nil + } else Nil + case _ => Nil + } + } else Nil + val recreated = settings.userSetSettings.toList.filterNot(_ eq settings.printArgs).flatMap(_.unparse) + val argsFile = (recreated ::: singleOuputDir ::: sources.map(_.file.absolute.toString())).mkString("", "\n", "\n") settings.printArgs.value match { case "-" => reporter.echo(argsFile) @@ -1474,7 +1499,7 @@ class Global(var currentSettings: Settings, reporter0: Reporter) private final val GlobalPhaseName = "global (synthetic)" protected final val totalCompileTime = statistics.newTimer("#total compile time", GlobalPhaseName) - def compileUnits(units: List[CompilationUnit], fromPhase: Phase): Unit = compileUnitsInternal(units,fromPhase) + def compileUnits(units: List[CompilationUnit], fromPhase: Phase): Unit = compileUnitsInternal(units,fromPhase) private def compileUnitsInternal(units: List[CompilationUnit], fromPhase: Phase) { units foreach addUnit reporter.reset() @@ -1701,6 +1726,13 @@ class Global(var currentSettings: Settings, reporter0: Reporter) } def createJavadoc = false + + final val closeableRegistry: CloseableRegistry = new CloseableRegistry + + def close(): Unit = { + perRunCaches.clearAll() + closeableRegistry.close() + } } object Global { diff --git a/src/compiler/scala/tools/nsc/PickleExtractor.scala b/src/compiler/scala/tools/nsc/PickleExtractor.scala new file mode 100644 index 000000000000..53a54b12e12a --- /dev/null +++ b/src/compiler/scala/tools/nsc/PickleExtractor.scala @@ -0,0 +1,121 @@ +package scala.tools.nsc + +import java.io.Closeable +import java.nio.file.attribute.BasicFileAttributes +import java.nio.file.{FileVisitResult, Files, Path, SimpleFileVisitor, _} + +import scala.collection.JavaConverters.{asScalaBufferConverter, bufferAsJavaListConverter, collectionAsScalaIterableConverter} +import scala.reflect.internal.pickling.ByteCodecs +import scala.reflect.io.RootPath +import scala.tools.asm.tree.ClassNode +import scala.tools.asm.{ClassReader, ClassWriter, Opcodes} + +object PickleExtractor { + + def main(args: Array[String]): Unit = { + args.toList match { + case input :: output :: Nil => + process(Paths.get(input), Paths.get(output)) + case _ => + } + } + def process(input: Path, output: Path): Unit = { + val inputPath = RootPath(input, writable = false) + val outputPath = RootPath(output, writable = true) + try { + val root = inputPath.root + Files.createDirectories(outputPath.root) + val visitor = new SimpleFileVisitor[Path] { + override def preVisitDirectory(dir: Path, attrs: BasicFileAttributes): FileVisitResult = { + if (dir != root) { + val outputDir = outputPath.root.resolve(root.relativize(dir).toString) + Files.createDirectories(outputDir) + } + FileVisitResult.CONTINUE + } + override def visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult = { + if (file.getFileName.toString.endsWith(".class")) { + stripClassFile(Files.readAllBytes(file)) match { + case Class(out) => + Files.write(outputPath.root.resolve(root.relativize(file).toString), out) + case Pickle(out) => + Files.write(outputPath.root.resolve(root.relativize(file).toString.replaceAll(".class$", ".sig")), out) + case Skip => + } + } + FileVisitResult.CONTINUE + } + } + Files.walkFileTree(root, visitor) + } finally { + inputPath.close() + outputPath.close() + } + } + + def stripClassFile(classfile: Array[Byte]): OutputFile = { + val input = new ClassNode() + new ClassReader(classfile).accept(input, ClassReader.SKIP_DEBUG | ClassReader.SKIP_FRAMES | ClassReader.SKIP_CODE) + var output = new ClassNode() + output.name = input.name + output.access = input.access + output.version = input.version + + var foundScalaSig = false + + def isScalaAnnotation(desc: String) = (desc == "Lscala/reflect/ScalaSignature;" || desc == "Lscala/reflect/ScalaLongSignature;") && { + foundScalaSig = true + + true + } + + var pickleData: Array[Byte] = null + if (input.visibleAnnotations != null) { + input.visibleAnnotations.asScala.foreach { node => + if (node.desc == "Lscala/reflect/ScalaSignature;") { + val Array("bytes", data: String) = node.values.toArray() + val bytes = data.getBytes(java.nio.charset.StandardCharsets.UTF_8) + val len = ByteCodecs.decode(bytes) + pickleData = bytes.take(len) + } else if (node.desc == "Lscala/reflect/ScalaLongSignature;") { + val Array("bytes", data: java.util.Collection[String @unchecked]) = node.values.toArray() + val encoded = data.asScala.toArray flatMap (_.getBytes(java.nio.charset.StandardCharsets.UTF_8)) + val len = ByteCodecs.decode(encoded) + pickleData = encoded.take(len) + } + } + output.visibleAnnotations = input.visibleAnnotations.asScala.filter(node => isScalaAnnotation(node.desc) && { + true + }).asJava + } + var foundScalaAttr = false + if (input.attrs != null) { + output.attrs = input.attrs.asScala.filter(attr => (attr.`type` == "Scala" || attr.`type` == "ScalaSig") && { + foundScalaAttr = true; + true + }).asJava + } + val writer = new ClassWriter(Opcodes.ASM7_EXPERIMENTAL) + val isScalaRaw = foundScalaAttr && !foundScalaSig + if (isScalaRaw) Skip + else { + if (pickleData == null) { + output = input + output.accept(writer) + Class(writer.toByteArray) + } else { + output.accept(writer) + Pickle(pickleData) + } + } + } + + sealed abstract class OutputFile + + case object Skip extends OutputFile + + case class Class(content: Array[Byte]) extends OutputFile + + case class Pickle(content: Array[Byte]) extends OutputFile + +} diff --git a/src/compiler/scala/tools/nsc/PipelineMain.scala b/src/compiler/scala/tools/nsc/PipelineMain.scala new file mode 100644 index 000000000000..84ed48a23300 --- /dev/null +++ b/src/compiler/scala/tools/nsc/PipelineMain.scala @@ -0,0 +1,682 @@ +/* NSC -- new Scala compiler + * Copyright 2005-2018 LAMP/EPFL + * @author Martin Odersky + */ +package scala.tools.nsc + +import java.io.File +import java.lang.Thread.UncaughtExceptionHandler +import java.nio.file.attribute.FileTime +import java.nio.file.{Files, Path, Paths} +import java.time.Instant +import java.util.Collections +import java.util.concurrent.atomic.AtomicInteger + +import javax.tools.ToolProvider + +import scala.collection.JavaConverters.asScalaIteratorConverter +import scala.collection.{immutable, mutable, parallel} +import scala.concurrent._ +import scala.concurrent.duration.Duration +import scala.reflect.internal.pickling.PickleBuffer +import scala.reflect.internal.util.FakePos +import scala.reflect.io.RootPath +import scala.tools.nsc.io.AbstractFile +import scala.tools.nsc.reporters.{ConsoleReporter, Reporter} +import scala.tools.nsc.util.ClassPath +import scala.util.{Failure, Success, Try} + +class PipelineMainClass(label: String, parallelism: Int, strategy: BuildStrategy, argFiles: Seq[Path], useJars: Boolean) { + private val pickleCacheConfigured = System.getProperty("scala.pipeline.picklecache") + private val pickleCache: Path = { + if (pickleCacheConfigured == null) Files.createTempDirectory("scala.picklecache") + else { + Paths.get(pickleCacheConfigured) + } + } + private def cachePath(file: Path): Path = { + val newExtension = if (useJars) ".jar" else "" + changeExtension(pickleCache.resolve("./" + file).normalize(), newExtension) + } + + private val strippedAndExportedClassPath = mutable.HashMap[Path, Path]() + + /** Forward errors to the (current) reporter. */ + protected def scalacError(msg: String): Unit = { + reporter.error(FakePos("scalac"), msg + "\n scalac -help gives more information") + } + + private var reporter: Reporter = _ + + private object handler extends UncaughtExceptionHandler { + override def uncaughtException(t: Thread, e: Throwable): Unit = { + e.printStackTrace() + System.exit(-1) + } + } + + implicit val executor = ExecutionContext.fromExecutor(new java.util.concurrent.ForkJoinPool(parallelism), t => handler.uncaughtException(Thread.currentThread(), t)) + val fileManager = ToolProvider.getSystemJavaCompiler.getStandardFileManager(null, null, null) + def changeExtension(p: Path, newExtension: String): Path = { + val fileName = p.getFileName.toString + val changedFileName = fileName.lastIndexOf('.') match { + case -1 => fileName + newExtension + case n => fileName.substring(0, n) + newExtension + } + p.getParent.resolve(changedFileName) + } + + def registerPickleClassPath[G <: Global](output: Path, data: mutable.AnyRefMap[G#Symbol, PickleBuffer]): Unit = { + val jarPath = cachePath(output) + val root = RootPath(jarPath, writable = true) + Files.createDirectories(root.root) + + val dirs = mutable.Map[G#Symbol, Path]() + def packageDir(packSymbol: G#Symbol): Path = { + if (packSymbol.isEmptyPackageClass) root.root + else if (dirs.contains(packSymbol)) dirs(packSymbol) + else if (packSymbol.owner.isRoot) { + val subDir = root.root.resolve(packSymbol.encodedName) + Files.createDirectories(subDir) + dirs.put(packSymbol, subDir) + subDir + } else { + val base = packageDir(packSymbol.owner) + val subDir = base.resolve(packSymbol.encodedName) + Files.createDirectories(subDir) + dirs.put(packSymbol, subDir) + subDir + } + } + val written = new java.util.IdentityHashMap[AnyRef, Unit]() + try { + for ((symbol, pickle) <- data) { + if (!written.containsKey(pickle)) { + val base = packageDir(symbol.owner) + val primary = base.resolve(symbol.encodedName + ".sig") + Files.write(primary, pickle.bytes) + written.put(pickle, ()) + } + } + } finally { + root.close() + } + Files.setLastModifiedTime(jarPath, FileTime.from(Instant.now())) + strippedAndExportedClassPath.put(output.toRealPath().normalize(), jarPath) + } + + + def writeDotFile(dependsOn: mutable.LinkedHashMap[Task, List[Dependency]]): Unit = { + val builder = new java.lang.StringBuilder() + builder.append("digraph projects {\n") + for ((p, deps) <- dependsOn) { + //builder.append(" node \"[]").append(p.label).append("\";\n") + for (dep <- deps) { + builder.append(" \"").append(p.label).append("\" -> \"").append(dep.t.label).append("\" [") + if (dep.isMacro) builder.append("label=M") + else if (dep.isPlugin) builder.append("label=P") + builder.append("];\n") + } + } + builder.append("}\n") + val path = Paths.get("projects.dot") + Files.write(path, builder.toString.getBytes(java.nio.charset.StandardCharsets.UTF_8)) + println("Wrote project dependency graph to: " + path.toAbsolutePath) + } + + private case class Dependency(t: Task, isMacro: Boolean, isPlugin: Boolean) + + def process(): Boolean = { + println(s"parallelism = $parallelism, strategy = $strategy") + + reporter = new ConsoleReporter(new Settings(scalacError)) + + def commandFor(argFileArg: Path): Task = { + val ss = new Settings(scalacError) + val command = new CompilerCommand(("@" + argFileArg) :: Nil, ss) + Task(argFileArg, command, command.files) + } + + val projects: List[Task] = argFiles.toList.map(commandFor) + val numProjects = projects.size + val produces = mutable.LinkedHashMap[Path, Task]() + for (p <- projects) { + produces(p.outputDir) = p + } + val dependsOn = mutable.LinkedHashMap[Task, List[Dependency]]() + for (p <- projects) { + val macroDeps = p.macroClassPath.flatMap(p => produces.get(p)).toList.filterNot(_ == p).map(t => Dependency(t, isMacro = true, isPlugin = false)) + val pluginDeps = p.pluginClassPath.flatMap(p => produces.get(p)).toList.filterNot(_ == p).map(t => Dependency(t, isMacro = false, isPlugin = true)) + val classPathDeps = p.classPath.flatMap(p => produces.get(p)).toList.filterNot(_ == p).filterNot(p => macroDeps.exists(_.t == p)).map(t => Dependency(t, isMacro = false, isPlugin = false)) + dependsOn(p) = classPathDeps ++ macroDeps ++ pluginDeps + } + val dependedOn: Set[Task] = dependsOn.valuesIterator.flatten.map(_.t).toSet + val externalClassPath = projects.iterator.flatMap(_.classPath).filter(p => !produces.contains(p) && Files.exists(p)).toSet + + if (strategy != Traditional) { + val exportTimer = new Timer + exportTimer.start() + for (entry <- externalClassPath) { + val extracted = cachePath(entry) + val sourceTimeStamp = Files.getLastModifiedTime(entry) + if (Files.exists(extracted) && Files.getLastModifiedTime(extracted) == sourceTimeStamp) { + // println(s"Skipped export of pickles from $entry to $extracted (up to date)") + } else { + PickleExtractor.process(entry, extracted) + Files.setLastModifiedTime(extracted, sourceTimeStamp) + println(s"Exported pickles from $entry to $extracted") + Files.setLastModifiedTime(extracted, sourceTimeStamp) + } + strippedAndExportedClassPath(entry) = extracted + } + exportTimer.stop() + println(f"Exported external classpath in ${exportTimer.durationMs}%.0f ms") + } + + writeDotFile(dependsOn) + + val timer = new Timer + timer.start() + + def awaitAll(fs: Seq[Future[_]]): Future[_] = { + val done = Promise[Any]() + val allFutures = projects.flatMap(_.futures) + val count = allFutures.size + val counter = new AtomicInteger(count) + val handler = (a: Try[_]) => a match { + case f @ Failure(_) => + done.complete(f) + case Success(_) => + val remaining = counter.decrementAndGet() + if (remaining == 0) done.success(()) + } + + allFutures.foreach(_.onComplete(handler)) + done.future + } + + def awaitDone(): Unit = { + val allFutures: immutable.Seq[Future[_]] = projects.flatMap(_.futures) + val numAllFutures = allFutures.size + val awaitAllFutures: Future[_] = awaitAll(allFutures) + val numTasks = awaitAllFutures + var lastNumCompleted = allFutures.count(_.isCompleted) + while (true) try { + Await.result(awaitAllFutures, Duration(60, "s")) + timer.stop() + val numCompleted = allFutures.count(_.isCompleted) + println(s"PROGRESS: $numCompleted / $numAllFutures") + return + } catch { + case _: TimeoutException => + val numCompleted = allFutures.count(_.isCompleted) + if (numCompleted == lastNumCompleted) { + println(s"STALLED: $numCompleted / $numAllFutures") + println("Outline/Scala/Javac") + projects.map { + p => + def toX(b: Future[_]): String = b.value match { case None => "-"; case Some(Success(_)) => "x"; case Some(Failure(_)) => "!" } + val s = List(p.outlineDoneFuture, p.groupsDoneFuture, p.javaDoneFuture).map(toX).mkString(" ") + println(s + " " + p.label) + } + } else { + println(s"PROGRESS: $numCompleted / $numAllFutures") + } + } + } + strategy match { + case OutlineTypePipeline => + projects.foreach { p => + val isLeaf = !dependedOn.contains(p) + val depsReady = Future.sequence(dependsOn.getOrElse(p, Nil).map { task => p.dependencyReadyFuture(task) }) + val f = if (isLeaf) { + for { + _ <- depsReady + _ <- { + p.outlineDone.complete(Success(())) + p.fullCompile() + Future.sequence(p.groups.map(_.done.future)) + } + } yield { + p.javaCompile() + } + } else { + for { + _ <- depsReady + _ <- { + p.outlineCompile() + p.outlineDone.future + } + _ <- { + p.fullCompile() + Future.sequence(p.groups.map(_.done.future)) + } + } yield { + p.javaCompile() + } + } + f.onComplete { _ => p.compiler.close() } + } + + awaitDone() + + for (p <- projects) { + val dependencies = dependsOn(p).map(_.t) + + def maxByOrZero[A](as: List[A])(f: A => Double): Double = if (as.isEmpty) 0d else as.map(f).max + + val maxOutlineCriticalPathMs = maxByOrZero(dependencies)(_.outlineCriticalPathMs) + p.outlineCriticalPathMs = maxOutlineCriticalPathMs + p.outlineTimer.durationMs + p.regularCriticalPathMs = maxOutlineCriticalPathMs + maxByOrZero(p.groups)(_.timer.durationMs) + p.fullCriticalPathMs = maxByOrZero(dependencies)(_.fullCriticalPathMs) + p.groups.map(_.timer.durationMs).sum + } + + if (parallelism == 1) { + val criticalPath = projects.maxBy(_.regularCriticalPathMs) + println(f"Critical path: ${criticalPath.regularCriticalPathMs}%.0f ms. Wall Clock: ${timer.durationMs}%.0f ms") + } else + println(f" Wall Clock: ${timer.durationMs}%.0f ms") + case Pipeline => + projects.foreach { p => + val depsReady = Future.sequence(dependsOn.getOrElse(p, Nil).map(task => p.dependencyReadyFuture(task))) + val f = for { + _ <- depsReady + _ <- { + val isLeaf = !dependedOn.contains(p) + if (isLeaf) { + p.outlineDone.complete(Success(())) + p.fullCompile() + } else + p.fullCompileExportPickles() + // Start javac after scalac has completely finished + Future.sequence(p.groups.map(_.done.future)) + } + } yield { + p.javaCompile() + } + f.onComplete { _ => p.compiler.close() } + } + awaitDone() + + for (p <- projects) { + val dependencies = dependsOn(p).map(_.t) + + def maxByOrZero[A](as: List[A])(f: A => Double): Double = if (as.isEmpty) 0d else as.map(f).max + + val maxOutlineCriticalPathMs = maxByOrZero(dependencies)(_.outlineCriticalPathMs) + p.outlineCriticalPathMs = maxOutlineCriticalPathMs + p.outlineTimer.durationMs + p.regularCriticalPathMs = maxOutlineCriticalPathMs + maxByOrZero(p.groups)(_.timer.durationMs) + p.fullCriticalPathMs = maxByOrZero(dependencies)(_.fullCriticalPathMs) + p.groups.map(_.timer.durationMs).sum + } + + if (parallelism == 1) { + val criticalPath = projects.maxBy(_.regularCriticalPathMs) + println(f"Critical path: ${criticalPath.regularCriticalPathMs}%.0f ms. Wall Clock: ${timer.durationMs}%.0f ms") + } else + println(f" Wall Clock: ${timer.durationMs}%.0f ms") + case Traditional => + projects.foreach { p => + val f1 = Future.sequence(dependsOn.getOrElse(p, Nil).map(_.t.javaDone.future)) + val f2 = f1.flatMap { _ => + p.outlineDone.complete(Success(())) + p.fullCompile() + Future.sequence(p.groups.map(_.done.future)).map(_ => p.javaCompile()) + } + f2.onComplete { _ => p.compiler.close() } + } + awaitDone() + + for (p <- projects) { + val dependencies = dependsOn(p).map(_.t) + + def maxByOrZero[A](as: List[A])(f: A => Double): Double = if (as.isEmpty) 0d else as.map(f).max + + p.fullCriticalPathMs = maxByOrZero(dependencies)(_.fullCriticalPathMs) + p.groups.map(_.timer.durationMs).sum + } + if (parallelism == 1) { + val maxFullCriticalPath: Double = projects.map(_.fullCriticalPathMs).max + println(f"Critical path: $maxFullCriticalPath%.0f ms. Wall Clock: ${timer.durationMs}%.0f ms") + } else { + println(f"Wall Clock: ${timer.durationMs}%.0f ms") + } + } + + writeChromeTrace(projects) + deleteTempPickleCache() + true + } + + private def deleteTempPickleCache(): Unit = { + if (pickleCacheConfigured == null) { + AbstractFile.getDirectory(pickleCache.toFile).delete() + } + } + + private def writeChromeTrace(projects: List[Task]) = { + val trace = new java.lang.StringBuilder() + trace.append("""{"traceEvents": [""") + val sb = new mutable.StringBuilder(trace) + + def durationEvent(name: String, cat: String, t: Timer): String = { + s"""{"name": "$name", "cat": "$cat", "ph": "X", "ts": ${(t.startMicros).toLong}, "dur": ${(t.durationMicros).toLong}, "pid": 0, "tid": ${t.thread.getId}}""" + } + + def projectEvents(p: Task): List[String] = { + val events = List.newBuilder[String] + if (p.outlineTimer.durationMicros > 0d) { + val desc = if (strategy == OutlineTypePipeline) "outline-type" else "parser-to-pickler" + events += durationEvent(p.label, desc, p.outlineTimer) + events += durationEvent(p.label, "pickle-export", p.pickleExportTimer) + } + for ((g, ix) <- p.groups.zipWithIndex) { + if (g.timer.durationMicros > 0d) + events += durationEvent(p.label, "compile-" + ix, g.timer) + } + if (p.javaTimer.durationMicros > 0d) { + val desc = "javac" + events += durationEvent(p.label, desc, p.javaTimer) + } + events.result() + } + + projects.iterator.flatMap(projectEvents).addString(sb, ",\n") + trace.append("]}") + val traceFile = Paths.get(s"build-${label}.trace") + Files.write(traceFile, trace.toString.getBytes()) + println("Chrome trace written to " + traceFile.toAbsolutePath) + } + + case class Group(files: List[String]) { + val timer = new Timer + val done = Promise[Unit]() + } + + private case class Task(argsFile: Path, command: CompilerCommand, files: List[String]) { + val label = argsFile.toString.replaceAll("target/", "").replaceAll("""(.*)/(.*).args""", "$1:$2") + override def toString: String = argsFile.toString + def outputDir: Path = command.settings.outputDirs.getSingleOutput.get.file.toPath.toAbsolutePath.normalize() + private def expand(s: command.settings.PathSetting): List[Path] = { + ClassPath.expandPath(s.value, expandStar = true).map(s => Paths.get(s).toAbsolutePath.normalize()) + } + lazy val classPath: Seq[Path] = expand(command.settings.classpath) + lazy val macroClassPath: Seq[Path] = expand(command.settings.YmacroClasspath) + lazy val macroClassPathSet: Set[Path] = macroClassPath.toSet + lazy val pluginClassPath: Set[Path] = { + def asPath(p: String) = ClassPath split p + + val paths = command.settings.plugin.value filter (_ != "") flatMap (s => asPath(s) map (s => Paths.get(s))) + paths.toSet + } + def dependencyReadyFuture(dependency: Dependency) = if (dependency.isMacro) { + log(s"dependency is on macro classpath, will wait for .class files: ${dependency.t.label}") + dependency.t.javaDone.future + } else if (dependency.isPlugin) { + log(s"dependency is on plugin classpath, will wait for .class files: ${dependency.t.label}") + dependency.t.javaDone.future + } else + dependency.t.outlineDone.future + + + val cacheMacro = java.lang.Boolean.getBoolean("scala.pipeline.cache.macro.classloader") + val cachePlugin = java.lang.Boolean.getBoolean("scala.pipeline.cache.plugin.classloader") + if (cacheMacro) + command.settings.YcacheMacroClassLoader.value = "always" + if (cachePlugin) + command.settings.YcachePluginClassLoader.value = "always" + + val groups: List[Group] = { + val isScalaLibrary = files.exists(_.endsWith("Predef.scala")) + if (strategy != OutlineTypePipeline || isScalaLibrary) { + Group(files) :: Nil + } else { + command.settings.classpath.value = command.settings.outputDirs.getSingleOutput.get.toString + File.pathSeparator + command.settings.classpath.value + val length = files.length + val groups = (length.toDouble / 128).toInt.max(1) + files.grouped((length.toDouble / groups).ceil.toInt.max(1)).toList.map(Group(_)) + } + } + command.settings.outputDirs.getSingleOutput.get.file.mkdirs() + + val isGrouped = groups.size > 1 + + val outlineTimer = new Timer() + val pickleExportTimer = new Timer + val javaTimer = new Timer() + + var outlineCriticalPathMs = 0d + var regularCriticalPathMs = 0d + var fullCriticalPathMs = 0d + val outlineDone: Promise[Unit] = Promise[Unit]() + val outlineDoneFuture = outlineDone.future + val javaDone: Promise[Unit] = Promise[Unit]() + val javaDoneFuture: Future[_] = javaDone.future + val groupsDoneFuture: Future[List[Unit]] = Future.sequence(groups.map(_.done.future)) + val futures: List[Future[_]] = { + outlineDone.future :: javaDone.future :: groups.map(_.done.future) + } + + val originalClassPath: String = command.settings.classpath.value + + lazy val compiler: Global = try { + val result = newCompiler(command.settings) + val reporter = result.reporter + if (reporter.hasErrors) + reporter.flush() + else if (command.shouldStopWithInfo) + reporter.echo(command.getInfoMessage(result)) + result + } catch { + case t: Throwable => + t.printStackTrace() + throw t + } + + def outlineCompile(): Unit = { + outlineTimer.start() + try { + log("scalac outline: start") + command.settings.Youtline.value = true + command.settings.stopAfter.value = List("pickler") + command.settings.Ymacroexpand.value = command.settings.MacroExpand.None + val run1 = new compiler.Run() + run1 compile files + registerPickleClassPath(command.settings.outputDirs.getSingleOutput.get.file.toPath, run1.symData) + outlineTimer.stop() + reporter.finish() + if (reporter.hasErrors) { + log("scalac outline: failed") + outlineDone.complete(Failure(new RuntimeException(label + ": compile failed: "))) + } else { + log(f"scala outline: done ${outlineTimer.durationMs}%.0f ms") + outlineDone.complete(Success(())) + } + } catch { + case t: Throwable => + t.printStackTrace() + outlineDone.complete(Failure(new RuntimeException(label + ": compile failed: "))) + } + } + + def fullCompile(): Unit = { + command.settings.Youtline.value = false + command.settings.stopAfter.value = Nil + command.settings.Ymacroexpand.value = command.settings.MacroExpand.Normal + + val groupCount = groups.size + for ((group, ix) <- groups.zipWithIndex) { + group.done.completeWith { + Future { + log(s"scalac (${ix + 1}/$groupCount): start") + group.timer.start() + val compiler2 = newCompiler(command.settings) + try { + val run2 = new compiler2.Run() + run2 compile group.files + compiler2.reporter.finish() + if (compiler2.reporter.hasErrors) { + group.done.complete(Failure(new RuntimeException(label + ": compile failed: "))) + } else { + group.done.complete(Success(())) + } + } finally { + compiler2.close() + group.timer.stop() + } + log(f"scalac (${ix + 1}/$groupCount): done ${group.timer.durationMs}%.0f ms") + } + } + } + } + + def fullCompileExportPickles(): Unit = { + assert(groups.size == 1) + val group = groups.head + log("scalac: start") + outlineTimer.start() + try { + val run2 = new compiler.Run() { + + override def advancePhase(): Unit = { + if (compiler.phase == this.picklerPhase) { + outlineTimer.stop() + log(f"scalac outline: done ${outlineTimer.durationMs}%.0f ms") + pickleExportTimer.start() + registerPickleClassPath(command.settings.outputDirs.getSingleOutput.get.file.toPath, symData) + pickleExportTimer.stop() + log(f"scalac: exported pickles ${pickleExportTimer.durationMs}%.0f ms") + outlineDone.complete(Success(())) + group.timer.start() + } + super.advancePhase() + } + } + + run2 compile group.files + compiler.reporter.finish() + group.timer.stop() + if (compiler.reporter.hasErrors) { + log("scalac: failed") + if (!outlineDone.isCompleted) + outlineDone.complete(Failure(new RuntimeException(label + ": compile failed: "))) + group.done.complete(Failure(new RuntimeException(label + ": compile failed: "))) + } else { + log(f"scalac: done ${group.timer.durationMs}%.0f ms") + // outlineDone.complete(Success(())) + group.done.complete(Success(())) + } + } catch { + case t: Throwable => + t.printStackTrace() + if (!outlineDone.isCompleted) + outlineDone.complete(Failure(new RuntimeException(label + ": compile failed: "))) + if (!group.done.isCompleted) + group.done.complete(Failure(new RuntimeException(label + ": compile failed: "))) + } + } + + def javaCompile(): Unit = { + val javaSources = files.filter(_.endsWith(".java")) + if (javaSources.nonEmpty) { + log("javac: start") + javaTimer.start() + javaDone.completeWith(Future { + val opts = java.util.Arrays.asList("-d", command.settings.outdir.value, "-cp", command.settings.outdir.value + File.pathSeparator + originalClassPath) + val compileTask = ToolProvider.getSystemJavaCompiler.getTask(null, null, null, opts, null, fileManager.getJavaFileObjects(javaSources.toArray: _*)) + compileTask.setProcessors(Collections.emptyList()) + compileTask.call() + javaTimer.stop() + log(f"javac: done ${javaTimer.durationMs}%.0f ms") + () + }) + } else { + javaDone.complete(Success(())) + } + } + def log(msg: String): Unit = println(this.label + ": " + msg) + } + + final class Timer() { + private var startNanos: Long = 0 + private var endNanos: Long = 0 + def start(): Unit = { + assert(startNanos == 0L) + startNanos = System.nanoTime + } + var thread: Thread = Thread.currentThread() + def stop(): Unit = { + thread = Thread.currentThread() + endNanos = System.nanoTime() + } + def startMs: Double = startNanos.toDouble / 1000 / 1000 + def durationMs: Double = { + val result = (endNanos - startNanos).toDouble / 1000 / 1000 + if (result < 0) + getClass + result + } + def startMicros: Double = startNanos.toDouble / 1000d + def durationMicros: Double = (endNanos - startNanos).toDouble / 1000d + } + + protected def newCompiler(settings: Settings): Global = { + if (strategy != Traditional) { + val classPath = ClassPath.expandPath(settings.classpath.value, expandStar = true) + val modifiedClassPath = classPath.map { entry => + val entryPath = Paths.get(entry) + if (Files.exists(entryPath)) + strippedAndExportedClassPath.getOrElse(entryPath.toRealPath().normalize(), entryPath).toString + else + entryPath + } + settings.classpath.value = modifiedClassPath.mkString(java.io.File.pathSeparator) + } + Global(settings) + } +} + +sealed abstract class BuildStrategy + +/** Outline type check to compute type signatures as pickles as an input to downstream compilation. */ +case object OutlineTypePipeline extends BuildStrategy + +case object Pipeline extends BuildStrategy + +/** Emit class files before triggering downstream compilation */ +case object Traditional extends BuildStrategy + +object PipelineMain { + def main(args: Array[String]): Unit = { + val strategies = List(OutlineTypePipeline, Pipeline, Traditional) + val strategy = strategies.find(_.productPrefix.equalsIgnoreCase(System.getProperty("scala.pipeline.strategy", "pipeline"))).get + val parallelism = java.lang.Integer.getInteger("scala.pipeline.parallelism", parallel.availableProcessors) + val useJars = java.lang.Boolean.getBoolean("scala.pipeline.use.jar") + val argFiles: Seq[Path] = args match { + case Array(path) if Files.isDirectory(Paths.get(path)) => + Files.walk(Paths.get(path)).iterator().asScala.filter(_.getFileName.toString.endsWith(".args")).toList + case _ => + args.map(Paths.get(_)) + } + val main = new PipelineMainClass("1", parallelism, strategy, argFiles, useJars) + val result = main.process() + if (!result) + System.exit(1) + else + System.exit(0) + } +} + +object PipelineMainTest { + def main(args: Array[String]): Unit = { + var i = 0 + val argsFiles = Files.walk(Paths.get("/code/guardian-frontend")).iterator().asScala.filter(_.getFileName.toString.endsWith(".args")).toList + for (_ <- 1 to 2; n <- List(parallel.availableProcessors); strat <- List(Pipeline)) { + i += 1 + val main = new PipelineMainClass(strat + "-" + i, n, strat, argsFiles, useJars = false) + println(s"====== ITERATION $i=======") + val result = main.process() + if (!result) + System.exit(1) + } + System.exit(0) + } +} diff --git a/src/compiler/scala/tools/nsc/backend/JavaPlatform.scala b/src/compiler/scala/tools/nsc/backend/JavaPlatform.scala index ff11f434710b..05396fc6ce73 100644 --- a/src/compiler/scala/tools/nsc/backend/JavaPlatform.scala +++ b/src/compiler/scala/tools/nsc/backend/JavaPlatform.scala @@ -27,7 +27,7 @@ trait JavaPlatform extends Platform { private[nsc] var currentClassPath: Option[ClassPath] = None private[nsc] def classPath: ClassPath = { - if (currentClassPath.isEmpty) currentClassPath = Some(new PathResolver(settings).result) + if (currentClassPath.isEmpty) currentClassPath = Some(new PathResolver(settings, global.closeableRegistry).result) currentClassPath.get } diff --git a/src/compiler/scala/tools/nsc/backend/Platform.scala b/src/compiler/scala/tools/nsc/backend/Platform.scala index a69e79d4c4f1..c48f9f079fe1 100644 --- a/src/compiler/scala/tools/nsc/backend/Platform.scala +++ b/src/compiler/scala/tools/nsc/backend/Platform.scala @@ -13,7 +13,10 @@ package scala.tools.nsc package backend +import java.nio.ByteBuffer + import io.AbstractFile +import scala.tools.nsc.classpath.AggregateClassPath import scala.tools.nsc.util.ClassPath /** The platform dependent pieces of Global. @@ -45,4 +48,3 @@ trait Platform { */ def needCompile(bin: AbstractFile, src: AbstractFile): Boolean } - diff --git a/src/compiler/scala/tools/nsc/classpath/AggregateClassPath.scala b/src/compiler/scala/tools/nsc/classpath/AggregateClassPath.scala index 68fb3000b8c0..ac4970f3cb6b 100644 --- a/src/compiler/scala/tools/nsc/classpath/AggregateClassPath.scala +++ b/src/compiler/scala/tools/nsc/classpath/AggregateClassPath.scala @@ -66,7 +66,6 @@ case class AggregateClassPath(aggregates: Seq[ClassPath]) extends ClassPath { override def asClassPathStrings: Seq[String] = aggregates.map(_.asClassPathString).distinct override def asSourcePathString: String = ClassPath.join(aggregates map (_.asSourcePathString): _*) - override private[nsc] def packages(inPackage: String): Seq[PackageEntry] = { val aggregatedPackages = aggregates.flatMap(_.packages(inPackage)).distinct aggregatedPackages diff --git a/src/compiler/scala/tools/nsc/classpath/ClassPathFactory.scala b/src/compiler/scala/tools/nsc/classpath/ClassPathFactory.scala index fa9166483594..f2fb2b0224d7 100644 --- a/src/compiler/scala/tools/nsc/classpath/ClassPathFactory.scala +++ b/src/compiler/scala/tools/nsc/classpath/ClassPathFactory.scala @@ -14,7 +14,7 @@ package scala.tools.nsc.classpath import scala.reflect.io.{AbstractFile, VirtualDirectory} import scala.reflect.io.Path.string2path -import scala.tools.nsc.Settings +import scala.tools.nsc.{CloseableRegistry, Settings} import FileUtils.AbstractFileOps import scala.tools.nsc.util.ClassPath @@ -22,11 +22,11 @@ import scala.tools.nsc.util.ClassPath * Provides factory methods for classpath. When creating classpath instances for a given path, * it uses proper type of classpath depending on a types of particular files containing sources or classes. */ -class ClassPathFactory(settings: Settings) { +class ClassPathFactory(settings: Settings, closeableRegistry: CloseableRegistry) { /** * Create a new classpath based on the abstract file. */ - def newClassPath(file: AbstractFile): ClassPath = ClassPathFactory.newClassPath(file, settings) + def newClassPath(file: AbstractFile): ClassPath = ClassPathFactory.newClassPath(file, settings, closeableRegistry) /** * Creators for sub classpaths which preserve this context. @@ -70,7 +70,7 @@ class ClassPathFactory(settings: Settings) { private def createSourcePath(file: AbstractFile): ClassPath = if (file.isJarOrZip) - ZipAndJarSourcePathFactory.create(file, settings) + ZipAndJarSourcePathFactory.create(file, settings, closeableRegistry) else if (file.isDirectory) DirectorySourcePath(file.file) else @@ -78,11 +78,11 @@ class ClassPathFactory(settings: Settings) { } object ClassPathFactory { - def newClassPath(file: AbstractFile, settings: Settings): ClassPath = file match { + def newClassPath(file: AbstractFile, settings: Settings, closeableRegistry: CloseableRegistry): ClassPath = file match { case vd: VirtualDirectory => VirtualDirectoryClassPath(vd) case _ => if (file.isJarOrZip) - ZipAndJarClassPathFactory.create(file, settings) + ZipAndJarClassPathFactory.create(file, settings, closeableRegistry) else if (file.isDirectory) DirectoryClassPath(file.file) else diff --git a/src/compiler/scala/tools/nsc/classpath/DirectoryClassPath.scala b/src/compiler/scala/tools/nsc/classpath/DirectoryClassPath.scala index 9f51672e79a6..9707af9b1d25 100644 --- a/src/compiler/scala/tools/nsc/classpath/DirectoryClassPath.scala +++ b/src/compiler/scala/tools/nsc/classpath/DirectoryClassPath.scala @@ -12,7 +12,7 @@ package scala.tools.nsc.classpath -import java.io.File +import java.io.{Closeable, File} import java.net.{URI, URL} import java.nio.file.{FileSystems, Files, SimpleFileVisitor} import java.util.function.IntFunction @@ -25,6 +25,7 @@ import FileUtils._ import scala.collection.JavaConverters._ import scala.collection.immutable import scala.reflect.internal.JDK9Reflectors +import scala.tools.nsc.CloseableRegistry import scala.tools.nsc.classpath.PackageNameUtils.{packageContains, separatePkgAndClassNames} /** @@ -61,6 +62,7 @@ trait DirectoryLookup[FileEntryType <: ClassRepresentation] extends ClassPath { private[nsc] def packages(inPackage: String): Seq[PackageEntry] = { val dirForPackage = getDirectory(inPackage) + val nestedDirs: Array[F] = dirForPackage match { case None => emptyFiles case Some(directory) => listChildren(directory, Some(isPackage)) @@ -122,7 +124,12 @@ trait JFileDirectoryLookup[FileEntryType <: ClassRepresentation] extends Directo // // Note this behaviour can be enabled in javac with `javac -XDsortfiles`, but that's only // intended to improve determinism of the compiler for compiler hackers. - util.Arrays.sort(listing, (o1: File, o2: File) => o1.getName.compareTo(o2.getName)) + util.Arrays.sort(listing, (o1: File, o2: File) => { + if (o1 == null || o1.getName == null || o2 == null || o2.getName == null) + getClass + o1.getName.compareTo(o2.getName) + } + ) listing } protected def getName(f: File): String = f.getName @@ -137,7 +144,7 @@ trait JFileDirectoryLookup[FileEntryType <: ClassRepresentation] extends Directo object JrtClassPath { import java.nio.file._, java.net.URI - def apply(release: Option[String]): Option[ClassPath] = { + def apply(release: Option[String], closeableRegistry: CloseableRegistry): Option[ClassPath] = { import scala.util.Properties._ if (!isJavaAtLeast("9")) None else { @@ -154,7 +161,11 @@ object JrtClassPath { try { val ctSym = Paths.get(javaHome).resolve("lib").resolve("ct.sym") if (Files.notExists(ctSym)) None - else Some(new CtSymClassPath(ctSym, v.toInt)) + else { + val classPath = new CtSymClassPath(ctSym, v.toInt) + closeableRegistry.registerClosable(classPath) + Some(classPath) + } } catch { case _: Throwable => None } @@ -230,7 +241,7 @@ final class JrtClassPath(fs: java.nio.file.FileSystem) extends ClassPath with No /** * Implementation `ClassPath` based on the $JAVA_HOME/lib/ct.sym backing http://openjdk.java.net/jeps/247 */ -final class CtSymClassPath(ctSym: java.nio.file.Path, release: Int) extends ClassPath with NoSourcePaths { +final class CtSymClassPath(ctSym: java.nio.file.Path, release: Int) extends ClassPath with NoSourcePaths with Closeable { import java.nio.file.Path, java.nio.file._ private val fileSystem: FileSystem = FileSystems.newFileSystem(ctSym, null) @@ -276,7 +287,7 @@ final class CtSymClassPath(ctSym: java.nio.file.Path, release: Int) extends Clas def asURLs: Seq[URL] = Nil def asClassPathStrings: Seq[String] = Nil - + override def close(): Unit = fileSystem.close() def findClassFile(className: String): Option[AbstractFile] = { if (!className.contains(".")) None else { diff --git a/src/compiler/scala/tools/nsc/classpath/FileUtils.scala b/src/compiler/scala/tools/nsc/classpath/FileUtils.scala index 059a83da796c..aa4d81736195 100644 --- a/src/compiler/scala/tools/nsc/classpath/FileUtils.scala +++ b/src/compiler/scala/tools/nsc/classpath/FileUtils.scala @@ -24,7 +24,7 @@ object FileUtils { implicit class AbstractFileOps(val file: AbstractFile) extends AnyVal { def isPackage: Boolean = file.isDirectory && mayBeValidPackage(file.name) - def isClass: Boolean = !file.isDirectory && file.hasExtension("class") + def isClass: Boolean = !file.isDirectory && (file.hasExtension("class") || file.hasExtension("sig")) def isScalaOrJavaSource: Boolean = !file.isDirectory && (file.hasExtension("scala") || file.hasExtension("java")) diff --git a/src/compiler/scala/tools/nsc/classpath/VirtualDirectoryClassPath.scala b/src/compiler/scala/tools/nsc/classpath/VirtualDirectoryClassPath.scala index 5b157e9b386e..e0e316513595 100644 --- a/src/compiler/scala/tools/nsc/classpath/VirtualDirectoryClassPath.scala +++ b/src/compiler/scala/tools/nsc/classpath/VirtualDirectoryClassPath.scala @@ -35,9 +35,8 @@ case class VirtualDirectoryClassPath(dir: VirtualDirectory) extends ClassPath wi def isPackage(f: AbstractFile): Boolean = f.isPackage // mimic the behavior of the old nsc.util.DirectoryClassPath - def asURLs: Seq[URL] = Seq(new URL(dir.name)) + def asURLs: Seq[URL] = Seq(new URL("file://_VIRTUAL_/" + dir.name)) def asClassPathStrings: Seq[String] = Seq(dir.path) - override def findClass(className: String): Option[ClassRepresentation] = findClassFile(className) map ClassFileEntryImpl def findClassFile(className: String): Option[AbstractFile] = { diff --git a/src/compiler/scala/tools/nsc/classpath/ZipAndJarFileLookupFactory.scala b/src/compiler/scala/tools/nsc/classpath/ZipAndJarFileLookupFactory.scala index 6f8b9a55c0cd..3a72804080ef 100644 --- a/src/compiler/scala/tools/nsc/classpath/ZipAndJarFileLookupFactory.scala +++ b/src/compiler/scala/tools/nsc/classpath/ZipAndJarFileLookupFactory.scala @@ -12,16 +12,19 @@ package scala.tools.nsc.classpath -import java.io.File +import java.io.{Closeable, File} import java.net.URL import java.nio.file.Files import java.nio.file.attribute.{BasicFileAttributes, FileTime} +import java.util.{Timer, TimerTask} +import java.util.concurrent.atomic.AtomicInteger import scala.annotation.tailrec import scala.reflect.io.{AbstractFile, FileZipArchive, ManifestResources} import scala.tools.nsc.util.{ClassPath, ClassRepresentation} -import scala.tools.nsc.Settings +import scala.tools.nsc.{CloseableRegistry, Settings} import FileUtils._ +import scala.reflect.internal.SymbolTable /** * A trait providing an optional cache for classpath entries obtained from zip and jar files. @@ -29,18 +32,19 @@ import FileUtils._ * when there are a lot of projects having a lot of common dependencies. */ sealed trait ZipAndJarFileLookupFactory { - private val cache = new FileBasedCache[ClassPath] - - def create(zipFile: AbstractFile, settings: Settings): ClassPath = { - if (settings.YdisableFlatCpCaching || zipFile.file == null) createForZipFile(zipFile, settings.releaseValue) - else createUsingCache(zipFile, settings) + private val cache = new FileBasedCache[ClassPath with Closeable] + + def create(zipFile: AbstractFile, settings: Settings, closeableRegistry: CloseableRegistry): ClassPath = { + if (settings.YdisableFlatCpCaching || zipFile.file == null) { + val result: ClassPath with Closeable = createForZipFile(zipFile, settings.releaseValue) + closeableRegistry.registerClosable(result) + result + } else { + cache.getOrCreate(List(zipFile.file.toPath), () => createForZipFile(zipFile, settings.releaseValue), closeableRegistry, checkStamps = true) + } } - protected def createForZipFile(zipFile: AbstractFile, release: Option[String]): ClassPath - - private def createUsingCache(zipFile: AbstractFile, settings: Settings): ClassPath = { - cache.getOrCreate(List(zipFile.file.toPath), () => createForZipFile(zipFile, settings.releaseValue)) - } + protected def createForZipFile(zipFile: AbstractFile, release: Option[String]): ClassPath with Closeable } /** @@ -75,7 +79,7 @@ object ZipAndJarClassPathFactory extends ZipAndJarFileLookupFactory { * with a particularly prepared scala-library.jar. It should have all classes listed in the manifest like e.g. this entry: * Name: scala/Function2$mcFJD$sp.class */ - private case class ManifestResourcesClassPath(file: ManifestResources) extends ClassPath with NoSourcePaths { + private case class ManifestResourcesClassPath(file: ManifestResources) extends ClassPath with NoSourcePaths with Closeable { override def findClassFile(className: String): Option[AbstractFile] = { val (pkg, simpleClassName) = PackageNameUtils.separatePkgAndClassNames(className) classes(pkg).find(_.name == simpleClassName).map(_.file) @@ -84,6 +88,7 @@ object ZipAndJarClassPathFactory extends ZipAndJarFileLookupFactory { override def asClassPathStrings: Seq[String] = Seq(file.path) override def asURLs: Seq[URL] = file.toURLs() + override def close(): Unit = file.close() import ManifestResourcesClassPath.PackageFileInfo import ManifestResourcesClassPath.PackageInfo @@ -152,7 +157,7 @@ object ZipAndJarClassPathFactory extends ZipAndJarFileLookupFactory { case class PackageInfo(packageName: String, subpackages: List[AbstractFile]) } - override protected def createForZipFile(zipFile: AbstractFile, release: Option[String]): ClassPath = + override protected def createForZipFile(zipFile: AbstractFile, release: Option[String]): ClassPath with Closeable = if (zipFile.file == null) createWithoutUnderlyingFile(zipFile) else ZipArchiveClassPath(zipFile.file, release) @@ -183,15 +188,50 @@ object ZipAndJarSourcePathFactory extends ZipAndJarFileLookupFactory { override protected def isRequiredFileType(file: AbstractFile): Boolean = file.isScalaOrJavaSource } - override protected def createForZipFile(zipFile: AbstractFile, release: Option[String]): ClassPath = ZipArchiveSourcePath(zipFile.file) + override protected def createForZipFile(zipFile: AbstractFile, release: Option[String]): ClassPath with Closeable = ZipArchiveSourcePath(zipFile.file) } final class FileBasedCache[T] { import java.nio.file.Path private case class Stamp(lastModified: FileTime, fileKey: Object) - private val cache = collection.mutable.Map.empty[Seq[Path], (Seq[Stamp], T)] + private case class Entry(stamps: Seq[Stamp], t: T) { + val referenceCount: AtomicInteger = new AtomicInteger(1) + } + private val cache = collection.mutable.Map.empty[Seq[Path], Entry] + + private def referenceCountDecrementer(e: Entry, paths: Seq[Path]): Closeable = new Closeable { + var closed = false + override def close(): Unit = { + if (!closed) { + closed = true + val count = e.referenceCount.decrementAndGet() + if (count == 0) { + e.t match { + case cl: Closeable => + FileBasedCache.timer match { + case Some(timer) => + val task = new TimerTask { + override def run(): Unit = { + cache.synchronized { + if (e.referenceCount.compareAndSet(0, -1)) { + cache.remove(paths) + cl.close() + } + } + } + } + timer.schedule(task, FileBasedCache.deferCloseMs.toLong) + case None => + cl.close() + } + case _ => + } + } + } + } + } - def getOrCreate(paths: Seq[Path], create: () => T): T = cache.synchronized { + def getOrCreate(paths: Seq[Path], create: () => T, closeableRegistry: CloseableRegistry, checkStamps: Boolean): T = cache.synchronized { val stamps = paths.map { path => val attrs = Files.readAttributes(path, classOf[BasicFileAttributes]) val lastModified = attrs.lastModifiedTime() @@ -201,10 +241,35 @@ final class FileBasedCache[T] { } cache.get(paths) match { - case Some((cachedStamps, cached)) if cachedStamps == stamps => cached + case Some(e@Entry(cachedStamps, cached)) => + if (!checkStamps || cachedStamps == stamps) { + // Cache hit + val count = e.referenceCount.incrementAndGet() + assert(count > 0, (stamps, count)) + closeableRegistry.registerClosable(referenceCountDecrementer(e, paths)) + cached + } else { + // Cache miss: we found an entry but the underlying files have been modified + cached match { + case c: Closeable => + if (e.referenceCount.get() == 0) { + c.close() + } else { + // TODO: What do do here? Maybe add to a list of closeables polled by a cleanup thread? + } + } + val value = create() + val entry = Entry(stamps, value) + cache.put(paths, entry) + closeableRegistry.registerClosable(referenceCountDecrementer(entry, paths)) + value + } case _ => + // Cache miss val value = create() - cache.put(paths, (stamps, value)) + val entry = Entry(stamps, value) + cache.put(paths, entry) + closeableRegistry.registerClosable(referenceCountDecrementer(entry, paths)) value } } @@ -215,3 +280,12 @@ final class FileBasedCache[T] { cache.clear() } } + +object FileBasedCache { + private val deferCloseMs = Integer.getInteger("scalac.filebasedcache.defer.close.ms", 1000) + private val timer: Option[Timer] = { + if (deferCloseMs > 0) + Some(new java.util.Timer(true)) + else None + } +} diff --git a/src/compiler/scala/tools/nsc/classpath/ZipArchiveFileLookup.scala b/src/compiler/scala/tools/nsc/classpath/ZipArchiveFileLookup.scala index 32ec4cde4485..c658d4c01664 100644 --- a/src/compiler/scala/tools/nsc/classpath/ZipArchiveFileLookup.scala +++ b/src/compiler/scala/tools/nsc/classpath/ZipArchiveFileLookup.scala @@ -12,7 +12,7 @@ package scala.tools.nsc.classpath -import java.io.File +import java.io.{Closeable, File} import java.net.URL import scala.collection.Seq import scala.reflect.io.AbstractFile @@ -25,7 +25,7 @@ import scala.tools.nsc.util.{ClassPath, ClassRepresentation} * It provides common logic for classes handling class and source files. * It's aware of things like e.g. META-INF directory which is correctly skipped. */ -trait ZipArchiveFileLookup[FileEntryType <: ClassRepresentation] extends ClassPath { +trait ZipArchiveFileLookup[FileEntryType <: ClassRepresentation] extends ClassPath with Closeable { val zipFile: File def release: Option[String] @@ -33,8 +33,8 @@ trait ZipArchiveFileLookup[FileEntryType <: ClassRepresentation] extends ClassPa override def asURLs: Seq[URL] = Seq(zipFile.toURI.toURL) override def asClassPathStrings: Seq[String] = Seq(zipFile.getPath) - private val archive = new FileZipArchive(zipFile, release) + override def close(): Unit = archive.close() override private[nsc] def packages(inPackage: String): Seq[PackageEntry] = { val prefix = PackageNameUtils.packagePrefix(inPackage) diff --git a/src/compiler/scala/tools/nsc/javac/JavaParsers.scala b/src/compiler/scala/tools/nsc/javac/JavaParsers.scala index 08468cb505b9..d87fa7e8da81 100644 --- a/src/compiler/scala/tools/nsc/javac/JavaParsers.scala +++ b/src/compiler/scala/tools/nsc/javac/JavaParsers.scala @@ -395,6 +395,7 @@ trait JavaParsers extends ast.parser.ParsersCommon with JavaScanners { in.nextToken() case STRICTFP => addAnnot(ScalaStrictFPAttr) + in.nextToken() case SYNCHRONIZED => in.nextToken() case _ => diff --git a/src/compiler/scala/tools/nsc/plugins/Plugin.scala b/src/compiler/scala/tools/nsc/plugins/Plugin.scala index b76f67ccf6aa..910b2b258d64 100644 --- a/src/compiler/scala/tools/nsc/plugins/Plugin.scala +++ b/src/compiler/scala/tools/nsc/plugins/Plugin.scala @@ -17,7 +17,9 @@ import scala.tools.nsc.io.Jar import scala.reflect.internal.util.ScalaClassLoader import scala.reflect.io.{Directory, File, Path} import java.io.InputStream +import java.net.URL +import scala.collection.JavaConverters._ import scala.collection.mutable import scala.tools.nsc.classpath.FileBasedCache import scala.util.{Failure, Success, Try} @@ -93,26 +95,7 @@ object Plugin { private val PluginXML = "scalac-plugin.xml" - private val pluginClassLoadersCache = new FileBasedCache[ScalaClassLoader]() - - /** Create a class loader with the specified locations plus - * the loader that loaded the Scala compiler. - * - * If the class loader has already been created before and the - * file stamps are the same, the previous loader is returned to - * mitigate the cost of dynamic classloading as it has been - * measured in https://github.com/scala/scala-dev/issues/458. - */ - private def loaderFor(locations: Seq[Path], disableCache: Boolean): ScalaClassLoader = { - def newLoader = () => { - val compilerLoader = classOf[Plugin].getClassLoader - val urls = locations map (_.toURL) - ScalaClassLoader fromURLs (urls, compilerLoader) - } - - if (disableCache || locations.exists(!Jar.isJarOrZip(_))) newLoader() - else pluginClassLoadersCache.getOrCreate(locations.map(_.jfile.toPath()), newLoader) - } + private[nsc] val pluginClassLoadersCache = new FileBasedCache[ScalaClassLoader.URLClassLoader]() /** Try to load a plugin description from the specified location. */ @@ -155,40 +138,26 @@ object Plugin { paths: List[List[Path]], dirs: List[Path], ignoring: List[String], - disableClassLoaderCache: Boolean): List[Try[AnyClass]] = + findPluginClassloader: (Seq[Path] => ClassLoader)): List[Try[AnyClass]] = { - // List[(jar, Try(descriptor))] in dir - def scan(d: Directory) = - d.files.toList sortBy (_.name) filter (Jar isJarOrZip _) map (j => (j, loadDescriptionFromJar(j))) - type PDResults = List[Try[(PluginDescription, ScalaClassLoader)]] - // scan plugin dirs for jars containing plugins, ignoring dirs with none and other jars - val fromDirs: PDResults = dirs filter (_.isDirectory) flatMap { d => - scan(d.toDirectory) collect { - case (j, Success(pd)) => Success((pd, loaderFor(Seq(j), disableClassLoaderCache))) + val fromLoaders = paths.map {path => + val loader = findPluginClassloader(path) + loader.getResource(PluginXML) match { + case null => Failure(new MissingPluginException(path)) + case url => + val inputStream = url.openStream + try { + Try((PluginDescription.fromXML(inputStream), loader)) + } finally { + inputStream.close() + } } } - // scan jar paths for plugins, taking the first plugin you find. - // a path element can be either a plugin.jar or an exploded dir. - def findDescriptor(ps: List[Path]) = { - def loop(qs: List[Path]): Try[PluginDescription] = qs match { - case Nil => Failure(new MissingPluginException(ps)) - case p :: rest => - if (p.isDirectory) loadDescriptionFromFile(p.toDirectory / PluginXML) orElse loop(rest) - else if (p.isFile) loadDescriptionFromJar(p.toFile) orElse loop(rest) - else loop(rest) - } - loop(ps) - } - val fromPaths: PDResults = paths map (p => (p, findDescriptor(p))) map { - case (p, Success(pd)) => Success((pd, loaderFor(p, disableClassLoaderCache))) - case (_, Failure(e)) => Failure(e) - } - val seen = mutable.HashSet[String]() - val enabled = (fromPaths ::: fromDirs) map { + val enabled = fromLoaders map { case Success((pd, loader)) if seen(pd.classname) => // a nod to scala/bug#7494, take the plugin classes distinctly Failure(new PluginLoadException(pd.name, s"Ignoring duplicate plugin ${pd.name} (${pd.classname})")) diff --git a/src/compiler/scala/tools/nsc/plugins/Plugins.scala b/src/compiler/scala/tools/nsc/plugins/Plugins.scala index bba855ba541a..0146a978540c 100644 --- a/src/compiler/scala/tools/nsc/plugins/Plugins.scala +++ b/src/compiler/scala/tools/nsc/plugins/Plugins.scala @@ -13,7 +13,14 @@ package scala.tools.nsc package plugins +import java.net.URL + +import scala.reflect.internal.util.ScalaClassLoader import scala.reflect.io.Path +import scala.tools.nsc +import scala.tools.nsc.io.Jar +import scala.tools.nsc.plugins.Plugin.pluginClassLoadersCache +import scala.tools.nsc.typechecker.Macros import scala.tools.nsc.util.ClassPath import scala.tools.util.PathResolver.Defaults @@ -37,7 +44,7 @@ trait Plugins { global: Global => def injectDefault(s: String) = if (s.isEmpty) Defaults.scalaPluginPath else s asPath(settings.pluginsDir.value) map injectDefault map Path.apply } - val maybes = Plugin.loadAllFrom(paths, dirs, settings.disable.value, settings.YcachePluginClassLoader.value == settings.CachePolicy.None.name) + val maybes = Plugin.loadAllFrom(paths, dirs, settings.disable.value, findPluginClassLoader(_)) val (goods, errors) = maybes partition (_.isSuccess) // Explicit parameterization of recover to avoid -Xlint warning about inferred Any errors foreach (_.recover[Any] { @@ -53,6 +60,38 @@ trait Plugins { global: Global => classes map (Plugin.instantiate(_, this)) } + /** + * Locate or create the classloader to load a compiler plugin with `classpath`. + * + * Subclasses may override to customise the behaviour. + * + * @param classpath + * @return + */ + protected def findPluginClassLoader(classpath: Seq[Path]): ClassLoader = { + val policy = settings.YcachePluginClassLoader.value + val disableCache = policy == settings.CachePolicy.None.name + def newLoader = () => { + val compilerLoader = classOf[Plugin].getClassLoader + val urls = classpath map (_.toURL) + ScalaClassLoader fromURLs (urls, compilerLoader) + } + + // Create a class loader with the specified locations plus + // the loader that loaded the Scala compiler. + // + // If the class loader has already been created before and the + // file stamps are the same, the previous loader is returned to + // mitigate the cost of dynamic classloading as it has been + // measured in https://github.com/scala/scala-dev/issues/458. + + if (disableCache) { + val loader = newLoader() + closeableRegistry.registerClosable(loader) + loader + } else pluginClassLoadersCache.getOrCreate(classpath.map(_.jfile.toPath()), newLoader, closeableRegistry, policy == settings.CachePolicy.LastModified.name) + } + protected lazy val roughPluginsList: List[Plugin] = loadRoughPluginsList() /** Load all available plugins. Skips plugins that @@ -123,4 +162,44 @@ trait Plugins { global: Global => (for (plug <- roughPluginsList ; help <- plug.optionsHelp) yield { "\nOptions for plugin '%s':\n%s\n".format(plug.name, help) }).mkString + + /** Obtains a `ClassLoader` instance used for macro expansion. + * + * By default a new `ScalaClassLoader` is created using the classpath + * from global and the classloader of self as parent. + * + * Mirrors with runtime definitions (e.g. Repl) need to adjust this method. + */ + protected[scala] def findMacroClassLoader(): ClassLoader = { + val classpath: Seq[URL] = if (settings.YmacroClasspath.isSetByUser) { + for { + file <- scala.tools.nsc.util.ClassPath.expandPath(settings.YmacroClasspath.value, true) + af <- Option(nsc.io.AbstractFile getDirectory file) + } yield af.file.toURI.toURL + } else global.classPath.asURLs + def newLoader: () => ScalaClassLoader.URLClassLoader = () => { + analyzer.macroLogVerbose("macro classloader: initializing from -cp: %s".format(classpath)) + ScalaClassLoader.fromURLs(classpath, getClass.getClassLoader) + } + + val policy = settings.YcacheMacroClassLoader.value + val disableCache = policy == settings.CachePolicy.None.name + if (disableCache) newLoader() + else { + import scala.tools.nsc.io.Jar + import scala.reflect.io.{AbstractFile, Path} + + val urlsAndFiles = classpath.map(u => u -> AbstractFile.getURL(u)) + val hasNullURL = urlsAndFiles.filter(_._2 eq null) + if (hasNullURL.nonEmpty) { + // TODO if the only null is jrt:// we can still cache + // TODO filter out classpath elements pointing to non-existing files before we get here, that's another source of null + analyzer.macroLogVerbose(s"macro classloader: caching is disabled because `AbstractFile.getURL` returned `null` for ${hasNullURL.map(_._1).mkString(", ")}.") + perRunCaches.recordClassloader(newLoader()) + } else { + val locations = urlsAndFiles.map(t => Path(t._2.file)) + Macros.macroClassLoadersCache.getOrCreate(locations.map(_.jfile.toPath()), newLoader, closeableRegistry, checkStamps = policy == settings.CachePolicy.LastModified.name) + } + } + } } diff --git a/src/compiler/scala/tools/nsc/profile/Profiler.scala b/src/compiler/scala/tools/nsc/profile/Profiler.scala index 87654e8e8baa..68cfab2f16e3 100644 --- a/src/compiler/scala/tools/nsc/profile/Profiler.scala +++ b/src/compiler/scala/tools/nsc/profile/Profiler.scala @@ -14,31 +14,41 @@ package scala.tools.nsc.profile import java.io.{FileWriter, PrintWriter} import java.lang.management.ManagementFactory +import java.nio.file.{Files, Paths} import java.util.ServiceLoader import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicInteger + import javax.management.openmbean.CompositeData import javax.management.{Notification, NotificationEmitter, NotificationListener} -import scala.tools.nsc.{Phase, Settings} +import scala.collection.mutable +import scala.collection.mutable.ArrayBuffer +import scala.reflect.internal.util.ChromeTrace +import scala.reflect.io.{AbstractFile, File} +import scala.tools.nsc.{Global, Phase, Settings} object Profiler { def apply(settings: Settings):Profiler = if (!settings.YprofileEnabled) NoOpProfiler else { - val reporter = if(settings.YprofileDestination.isSetByUser) - new StreamProfileReporter(new PrintWriter(new FileWriter(settings.YprofileDestination.value, true))) - else ConsoleProfileReporter + val reporter = settings.YprofileDestination.value match { + case _ if !settings.YprofileDestination.isSetByUser => NoOpProfileReporter + case "-" => ConsoleProfileReporter + case path => new StreamProfileReporter(new PrintWriter(new FileWriter(path, true))) + } new RealProfiler(reporter, settings) } - private[profile] val emptySnap = ProfileSnap(0, "", 0, 0, 0, 0, 0, 0) + private[profile] val emptySnap = ProfileSnap(0, "", 0, 0, 0, 0, 0, 0, 0, 0) +} +case class GcEventData(pool:String, reportTimeNs: Long, gcStartMillis:Long, gcEndMillis:Long, durationMillis: Long, name:String, action:String, cause:String, threads:Long) { + val endNanos = System.nanoTime() } -case class GcEventData(pool:String, reportTimeNs: Long, gcStartMillis:Long, gcEndMillis:Long, name:String, action:String, cause:String, threads:Long) case class ProfileSnap(threadId: Long, threadName: String, snapTimeNanos : Long, - idleTimeNanos:Long, cpuTimeNanos: Long, userTimeNanos: Long, - allocatedBytes:Long, heapBytes:Long) { + idleTimeNanos:Long, cpuTimeNanos: Long, userTimeNanos: Long, + allocatedBytes:Long, heapBytes:Long, totalClassesLoaded: Long, totalJITCompilationTime: Long) { def updateHeap(heapBytes:Long) = { copy(heapBytes = heapBytes) } @@ -73,13 +83,29 @@ case class ProfileRange(start: ProfileSnap, end:ProfileSnap, phase:Phase, purpos def retainedHeapMB = toMegaBytes(end.heapBytes - start.heapBytes) } -sealed trait Profiler { +sealed abstract class Profiler { def finished(): Unit def beforePhase(phase: Phase): ProfileSnap def afterPhase(phase: Phase, profileBefore: ProfileSnap): Unit + + def beforeUnit(phase: Phase, file: AbstractFile): Unit + + def afterUnit(phase: Phase, file: AbstractFile): Unit + + def beforeTypedImplDef(sym: Global#Symbol): Unit = () + def afterTypedImplDef(sym: Global#Symbol): Unit = () + + def beforeImplicitSearch(pt: Global#Type): Unit = () + def afterImplicitSearch(pt: Global#Type): Unit = () + + def beforeMacroExpansion(macroSym: Global#Symbol): Unit = () + def afterMacroExpansion(macroSym: Global#Symbol): Unit = () + + def beforeCompletion(root: Global#Symbol, associatedFile: AbstractFile): Unit = () + def afterCompletion(root: Global#Symbol, associatedFile: AbstractFile): Unit = () } private [profile] object NoOpProfiler extends Profiler { @@ -87,6 +113,8 @@ private [profile] object NoOpProfiler extends Profiler { override def afterPhase(phase: Phase, profileBefore: ProfileSnap): Unit = () + override def beforeUnit(phase: Phase, file: AbstractFile): Unit = () + override def afterUnit(phase: Phase, file: AbstractFile): Unit = () override def finished(): Unit = () } private [profile] object RealProfiler { @@ -99,17 +127,55 @@ private [profile] object RealProfiler { val threadMx = ExtendedThreadMxBean.proxy if (threadMx.isThreadCpuTimeSupported) threadMx.setThreadCpuTimeEnabled(true) private val idGen = new AtomicInteger() + lazy val allPlugins = ServiceLoader.load(classOf[ProfilerPlugin]).iterator().asScala.toList + + private[profile] def snapThread( idleTimeNanos:Long): ProfileSnap = { + val current = Thread.currentThread() + val allocatedBytes = threadMx.getThreadAllocatedBytes(Thread.currentThread().getId) + ProfileSnap( + threadId = current.getId, + threadName = current.getName, + snapTimeNanos = System.nanoTime(), + idleTimeNanos = idleTimeNanos, + cpuTimeNanos = threadMx.getCurrentThreadCpuTime, + userTimeNanos = threadMx.getCurrentThreadUserTime, + allocatedBytes = allocatedBytes, + heapBytes = readHeapUsage(), + totalClassesLoaded = classLoaderMx.getTotalLoadedClassCount, + totalJITCompilationTime = compileMx.getTotalCompilationTime + ) + } + private def readHeapUsage() = RealProfiler.memoryMx.getHeapMemoryUsage.getUsed } private [profile] class RealProfiler(reporter : ProfileReporter, val settings: Settings) extends Profiler with NotificationListener { + private val mainThread = Thread.currentThread() + val id = RealProfiler.idGen.incrementAndGet() + object Category { + final val Run = "run" + final val Phase = "phase" + final val File = "file" + final val TypeCheck = "typecheck" + final val Implicit = "implicit" + final val Macro = "macro" + final val Completion = "completion" + } + + private val chromeTrace = { + if (settings.YprofileTrace.isSetByUser) + new ChromeTrace(Paths.get(settings.YprofileTrace.value)) + else null + } + if (chromeTrace != null) + chromeTrace.traceDurationEventStart(Category.Run, "scalac-" + id) + def completeBackground(threadRange: ProfileRange): Unit = { reporter.reportBackground(this, threadRange) } def outDir = settings.outputDirs.getSingleOutput.getOrElse(settings.outputDirs.outputs.head._2.file).toString - val id = RealProfiler.idGen.incrementAndGet() RealProfiler.gcMx foreach { case emitter: NotificationEmitter => emitter.addNotificationListener(this, null, null) case gc => println(s"Cant connect gcListener to ${gc.getClass}") @@ -117,25 +183,6 @@ private [profile] class RealProfiler(reporter : ProfileReporter, val settings: S val active = RealProfiler.allPlugins map (_.generate(this, settings)) - private val mainThread = Thread.currentThread() - - private[profile] def snapThread( idleTimeNanos:Long): ProfileSnap = { - import RealProfiler._ - val current = Thread.currentThread() - - ProfileSnap( - threadId = current.getId, - threadName = current.getName, - snapTimeNanos = System.nanoTime(), - idleTimeNanos = idleTimeNanos, - cpuTimeNanos = threadMx.getCurrentThreadCpuTime, - userTimeNanos = threadMx.getCurrentThreadUserTime, - allocatedBytes = threadMx.getThreadAllocatedBytes(Thread.currentThread().getId), - heapBytes = readHeapUsage() - ) - } - private def readHeapUsage() = RealProfiler.memoryMx.getHeapMemoryUsage.getUsed - private def doGC: Unit = { System.gc() System.runFinalization() @@ -151,8 +198,19 @@ private [profile] class RealProfiler(reporter : ProfileReporter, val settings: S case gc => } reporter.close(this) + if (chromeTrace != null) { + for (gcEvent <- gcEvents) { + val durationNanos = TimeUnit.MILLISECONDS.toNanos(gcEvent.durationMillis) + val startNanos = gcEvent.endNanos - durationNanos + chromeTrace.traceDurationEvent(gcEvent.name, startNanos, durationNanos, GcThreadId) + } + chromeTrace.traceDurationEventEnd(Category.Run, "scalac-" + id) + chromeTrace.close() + } } + private val gcEvents = ArrayBuffer[GcEventData]() + private val GcThreadId = "GC" override def handleNotification(notification: Notification, handback: scala.Any): Unit = { import java.lang.{Long => jLong} @@ -173,13 +231,30 @@ private [profile] class RealProfiler(reporter : ProfileReporter, val settings: S val startTime = info.get("startTime").asInstanceOf[jLong].longValue() val endTime = info.get("endTime").asInstanceOf[jLong].longValue() val threads = info.get("GcThreadCount").asInstanceOf[jInt].longValue() - reporter.reportGc(GcEventData("", reportNs, startTime, endTime, name, action, cause, threads)) + val gcEvent = GcEventData("", reportNs, startTime, endTime, duration, name, action, cause, threads) + synchronized { + gcEvents += gcEvent + } + reporter.reportGc(gcEvent) + } + } + + override def beforePhase(phase: Phase): ProfileSnap = { + assert(mainThread eq Thread.currentThread()) + if (chromeTrace != null) chromeTrace.traceDurationEventStart(Category.Phase, phase.name) + if (settings.YprofileRunGcBetweenPhases.containsPhase(phase)) + doGC + if (settings.YprofileExternalTool.containsPhase(phase)) { + println("Profile hook start") + ExternalToolHook.before() } + active foreach {_.beforePhase(phase)} + RealProfiler.snapThread(0) } override def afterPhase(phase: Phase, snapBefore: ProfileSnap): Unit = { assert(mainThread eq Thread.currentThread()) - val initialSnap = snapThread(0) + val initialSnap = RealProfiler.snapThread(0) active foreach {_.afterPhase(phase)} if (settings.YprofileExternalTool.containsPhase(phase)) { println("Profile hook stop") @@ -187,24 +262,85 @@ private [profile] class RealProfiler(reporter : ProfileReporter, val settings: S } val finalSnap = if (settings.YprofileRunGcBetweenPhases.containsPhase(phase)) { doGC - initialSnap.updateHeap(readHeapUsage()) + initialSnap.updateHeap(RealProfiler.readHeapUsage()) } else initialSnap + if (chromeTrace != null) chromeTrace.traceDurationEventEnd(Category.Phase, phase.name) reporter.reportForeground(this, ProfileRange(snapBefore, finalSnap, phase, "", 0, Thread.currentThread)) } - override def beforePhase(phase: Phase): ProfileSnap = { + override def beforeUnit(phase: Phase, file: AbstractFile): Unit = { assert(mainThread eq Thread.currentThread()) - if (settings.YprofileRunGcBetweenPhases.containsPhase(phase)) - doGC - if (settings.YprofileExternalTool.containsPhase(phase)) { - println("Profile hook start") - ExternalToolHook.before() + if (chromeTrace != null) chromeTrace.traceDurationEventStart(Category.File, file.name) + } + + private var nextAfterUnitSnap: Long = System.nanoTime() + + override def afterUnit(phase: Phase, file: AbstractFile): Unit = { + assert(mainThread eq Thread.currentThread()) + if (chromeTrace != null) { + val now = System.nanoTime() + chromeTrace.traceDurationEventEnd(Category.File, file.name) + if (now > nextAfterUnitSnap) { + val initialSnap = RealProfiler.snapThread(0) + chromeTrace.traceCounterEvent("allocBytes", "allocBytes", initialSnap.allocatedBytes, processWide = false) + chromeTrace.traceCounterEvent("heapBytes", "heapBytes", initialSnap.heapBytes, processWide = true) + chromeTrace.traceCounterEvent("classesLoaded", "classesLoaded", initialSnap.totalClassesLoaded, processWide = true) + chromeTrace.traceCounterEvent("jitCompilationTime", "jitCompilationTime", initialSnap.totalJITCompilationTime, processWide = true) + chromeTrace.traceCounterEvent("userTime", "userTime", initialSnap.userTimeNanos, processWide = false) + chromeTrace.traceCounterEvent("cpuTime", "cpuTime", initialSnap.cpuTimeNanos, processWide = false) + chromeTrace.traceCounterEvent("idleTime", "idleTime", initialSnap.idleTimeNanos, processWide = false) + nextAfterUnitSnap = System.nanoTime() + 10 * 1000 * 1000 + } } - active foreach {_.beforePhase(phase)} - snapThread(0) } + override def beforeTypedImplDef(sym: Global#Symbol): Unit = { + if (chromeTrace != null) chromeTrace.traceDurationEventStart(Category.TypeCheck, sym.rawname.toString) + } + override def afterTypedImplDef(sym: Global#Symbol): Unit = { + if (chromeTrace != null) chromeTrace.traceDurationEventEnd(Category.TypeCheck, sym.rawname.toString) + } + + override def beforeImplicitSearch(pt: Global#Type): Unit = { + if (chromeTrace != null) chromeTrace.traceDurationEventStart(Category.Implicit, "?[" + pt.typeSymbol.rawname + "]", colour = "yellow") + } + + override def afterImplicitSearch(pt: Global#Type): Unit = { + if (chromeTrace != null) chromeTrace.traceDurationEventEnd(Category.Implicit, "?[" + pt.typeSymbol.rawname + "]", colour = "yellow") + } + + override def beforeMacroExpansion(macroSym: Global#Symbol): Unit = { + if (chromeTrace != null) chromeTrace.traceDurationEventStart(Category.Macro, "«" + macroSym.rawname + "»", colour = "olive") + } + + override def afterMacroExpansion(macroSym: Global#Symbol): Unit = { + if (chromeTrace != null) chromeTrace.traceDurationEventEnd(Category.Macro, "«" + macroSym.rawname + "»", colour = "olive") + } + + override def beforeCompletion(root: Global#Symbol, associatedFile: AbstractFile): Unit = { + if (chromeTrace != null) { + chromeTrace.traceDurationEventStart(Category.Completion, "↯", colour = "thread_state_sleeping") + chromeTrace.traceDurationEventStart(Category.File, associatedFile.name) + chromeTrace.traceDurationEventStart(Category.Completion, completionName(root, associatedFile)) + } + } + + override def afterCompletion(root: Global#Symbol, associatedFile: AbstractFile): Unit = { + if (chromeTrace != null) { + chromeTrace.traceDurationEventEnd(Category.Completion, completionName(root, associatedFile)) + chromeTrace.traceDurationEventEnd(Category.File, associatedFile.name) + chromeTrace.traceDurationEventEnd(Category.Completion, "↯", colour = "thread_state_sleeping") + } + } + + private def completionName(root: Global#Symbol, associatedFile: AbstractFile): String = { + if (root.hasPackageFlag || root.isTopLevel) root.javaBinaryNameString + else { + val enclosing = root.enclosingTopLevelClass + enclosing.javaBinaryNameString + "::" + root.rawname.toString + } + } } object EventType extends Enumeration { @@ -228,24 +364,23 @@ sealed trait ProfileReporter { } object ConsoleProfileReporter extends ProfileReporter { + private val outWriter = new PrintWriter(Console.out) + private val delegate = new StreamProfileReporter(new PrintWriter(Console.out)) + override def reportBackground(profiler: RealProfiler, threadRange: ProfileRange): Unit = delegate.reportBackground(profiler, threadRange) + override def reportForeground(profiler: RealProfiler, threadRange: ProfileRange): Unit = delegate.reportForeground(profiler, threadRange) + override def close(profiler: RealProfiler): Unit = outWriter.flush() + + override def header(profiler: RealProfiler): Unit = delegate.header(profiler) + override def reportGc(data: GcEventData): Unit = delegate.reportGc(data) +} - - override def reportBackground(profiler: RealProfiler, threadRange: ProfileRange): Unit = - // TODO - ??? - override def reportForeground(profiler: RealProfiler, threadRange: ProfileRange): Unit = - // TODO - ??? - +object NoOpProfileReporter extends ProfileReporter { + override def reportBackground(profiler: RealProfiler, threadRange: ProfileRange): Unit = () + override def reportForeground(profiler: RealProfiler, threadRange: ProfileRange): Unit = () override def close(profiler: RealProfiler): Unit = () - override def header(profiler: RealProfiler): Unit = { - println(s"Profiler start (${profiler.id}) ${profiler.outDir}") - } - - override def reportGc(data: GcEventData): Unit = { - println(f"Profiler GC reported ${data.gcEndMillis - data.gcStartMillis}ms") - } + override def header(profiler: RealProfiler): Unit = () + override def reportGc(data: GcEventData): Unit = () } class StreamProfileReporter(out:PrintWriter) extends ProfileReporter { @@ -271,10 +406,8 @@ class StreamProfileReporter(out:PrintWriter) extends ProfileReporter { out.println(s"${EventType.GC},$start,${data.reportTimeNs},${data.gcStartMillis}, ${data.gcEndMillis},${data.name},${data.action},${data.cause},${data.threads}") } - override def close(profiler: RealProfiler): Unit = { - out.flush - out.close + out.flush() + out.close() } } - diff --git a/src/compiler/scala/tools/nsc/profile/ThreadPoolFactory.scala b/src/compiler/scala/tools/nsc/profile/ThreadPoolFactory.scala index 822a7317d284..641526a1de48 100644 --- a/src/compiler/scala/tools/nsc/profile/ThreadPoolFactory.scala +++ b/src/compiler/scala/tools/nsc/profile/ThreadPoolFactory.scala @@ -98,9 +98,9 @@ object ThreadPoolFactory { val data = new ThreadProfileData localData.set(data) - val profileStart = profiler.snapThread(0) + val profileStart = RealProfiler.snapThread(0) try worker.run finally { - val snap = profiler.snapThread(data.idleNs) + val snap = RealProfiler.snapThread(data.idleNs) val threadRange = ProfileRange(profileStart, snap, phase, shortId, data.taskCount, Thread.currentThread()) profiler.completeBackground(threadRange) } diff --git a/src/compiler/scala/tools/nsc/settings/AbsSettings.scala b/src/compiler/scala/tools/nsc/settings/AbsSettings.scala index 64eeb8717a9b..500f4bea606d 100644 --- a/src/compiler/scala/tools/nsc/settings/AbsSettings.scala +++ b/src/compiler/scala/tools/nsc/settings/AbsSettings.scala @@ -25,7 +25,7 @@ trait AbsSettings extends scala.reflect.internal.settings.AbsSettings { protected def allSettings: scala.collection.Set[Setting] // settings minus internal usage settings - def visibleSettings = allSettings filterNot (_.isInternalOnly) + def visibleSettings = allSettings.iterator filterNot (_.isInternalOnly) // only settings which differ from default def userSetSettings = visibleSettings filterNot (_.isDefault) diff --git a/src/compiler/scala/tools/nsc/settings/ScalaSettings.scala b/src/compiler/scala/tools/nsc/settings/ScalaSettings.scala index 655c3528d18f..7b2c441fc522 100644 --- a/src/compiler/scala/tools/nsc/settings/ScalaSettings.scala +++ b/src/compiler/scala/tools/nsc/settings/ScalaSettings.scala @@ -243,6 +243,9 @@ trait ScalaSettings extends AbsScalaSettings val YcacheMacroClassLoader = CachePolicy.setting("macro", "macros") val YpartialUnification = BooleanSetting ("-Ypartial-unification", "Enable partial unification in type constructor inference") val Yvirtpatmat = BooleanSetting ("-Yvirtpatmat", "Enable pattern matcher virtualization") + val Youtline = BooleanSetting ("-Youtline", "Don't compile method bodies. Use together with `-Ystop-afer:pickler to generate the pickled signatures for all source files.") + + val YmacroClasspath = PathSetting ("-Ymacro-classpath", "The classpath used to reflectively load macro implementations", "") val exposeEmptyPackage = BooleanSetting ("-Yexpose-empty-package", "Internal only: expose the empty package.").internalOnly() val Ydelambdafy = ChoiceSetting ("-Ydelambdafy", "strategy", "Strategy used for translating lambdas into JVM code.", List("inline", "method"), "method") @@ -257,9 +260,10 @@ trait ScalaSettings extends AbsScalaSettings def setting(style: String, styleLong: String) = ChoiceSetting(s"-Ycache-$style-class-loader", "policy", s"Policy for caching class loaders for $styleLong that are dynamically loaded.", values.map(_.name), None.name, values.map(_.help)) object None extends CachePolicy("none", "Don't cache class loader") object LastModified extends CachePolicy("last-modified", "Cache class loader, using file last-modified time to invalidate") + object Always extends CachePolicy("always", "Cache class loader with no invalidation") // TODO Jorge to add new policy. Think about whether there is a benefit to the user on offering this as a separate policy or unifying with the previous one. // object ZipMetadata extends CachePolicy("zip-metadata", "Cache classloade, using file last-modified time, then ZIP file metadata to invalidate") - def values: List[CachePolicy] = List(None, LastModified) + def values: List[CachePolicy] = List(None, LastModified, Always) } object optChoices extends MultiChoiceEnumeration { @@ -415,7 +419,9 @@ trait ScalaSettings extends AbsScalaSettings override def YhotStatisticsEnabled = YhotStatistics.value val YprofileEnabled = BooleanSetting("-Yprofile-enabled", "Enable profiling.") - val YprofileDestination = StringSetting("-Yprofile-destination", "file", "where to send profiling output - specify a file, default is to the console.", ""). + val YprofileDestination = StringSetting("-Yprofile-destination", "file", "Profiling output - specify a file or `-` for console.", ""). + withPostSetHook( _ => YprofileEnabled.value = true ) + val YprofileTrace = StringSetting("-Yprofile-trace", "file", "Capture trace of compilation in Chrome Trace format", "profile.trace"). withPostSetHook( _ => YprofileEnabled.value = true ) val YprofileExternalTool = PhasesSetting("-Yprofile-external-tool", "Enable profiling for a phase using an external tool hook. Generally only useful for a single phase", "typer"). withPostSetHook( _ => YprofileEnabled.value = true ) diff --git a/src/compiler/scala/tools/nsc/symtab/SymbolLoaders.scala b/src/compiler/scala/tools/nsc/symtab/SymbolLoaders.scala index 6444823efced..89d663b84b4c 100644 --- a/src/compiler/scala/tools/nsc/symtab/SymbolLoaders.scala +++ b/src/compiler/scala/tools/nsc/symtab/SymbolLoaders.scala @@ -203,6 +203,7 @@ abstract class SymbolLoaders { protected def doComplete(root: Symbol): Unit def sourcefile: Option[AbstractFile] = None + def associatedFile(self: Symbol): AbstractFile = NoAbstractFile /** * Description of the resource (ClassPath, AbstractFile) @@ -221,23 +222,29 @@ abstract class SymbolLoaders { } override def complete(root: Symbol) { + val assocFile = associatedFile(root) + currentRunProfilerBeforeCompletion(root, assocFile) try { - val start = java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(System.nanoTime()) - val currentphase = phase - doComplete(root) - phase = currentphase - informTime("loaded " + description, start) - ok = true - setSource(root) - setSource(root.companionSymbol) // module -> class, class -> module - } - catch { - case ex @ (_: IOException | _: MissingRequirementError) => - ok = false - signalError(root, ex) + try { + val start = java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(System.nanoTime()) + val currentphase = phase + doComplete(root) + phase = currentphase + informTime("loaded " + description, start) + ok = true + setSource(root) + setSource(root.companionSymbol) // module -> class, class -> module + } + catch { + case ex@(_: IOException | _: MissingRequirementError) => + ok = false + signalError(root, ex) + } + initRoot(root) + if (!root.isPackageClass) initRoot(root.companionSymbol) + } finally { + currentRunProfilerAfterCompletion(root, assocFile) } - initRoot(root) - if (!root.isPackageClass) initRoot(root.companionSymbol) } override def load(root: Symbol) { complete(root) } @@ -277,8 +284,11 @@ abstract class SymbolLoaders { val classPathEntries = classPath.list(packageName) - if (!root.isRoot) - for (entry <- classPathEntries.classesAndSources) initializeFromClassPath(root, entry) + if (!root.isRoot) { + val listing = classPathEntries.classesAndSources + for (entry <- listing) + initializeFromClassPath(root, entry) + } if (!root.isEmptyPackageClass) { for (pkg <- classPathEntries.packages) { val fullName = pkg.name @@ -336,18 +346,27 @@ abstract class SymbolLoaders { if (StatisticsStatics.areSomeColdStatsEnabled) statistics.stopTimer(statistics.classReadNanos, start) } override def sourcefile: Option[AbstractFile] = classfileParser.srcfile + override def associatedFile(self: Symbol): AbstractFile = classfile } class SourcefileLoader(val srcfile: AbstractFile) extends SymbolLoader with FlagAssigningCompleter { protected def description = "source file "+ srcfile.toString override def fromSource = true override def sourcefile = Some(srcfile) + override def associatedFile(self: Symbol): AbstractFile = srcfile protected def doComplete(root: Symbol): Unit = compileLate(srcfile) } object moduleClassLoader extends SymbolLoader with FlagAssigningCompleter { protected def description = "module class loader" protected def doComplete(root: Symbol) { root.sourceModule.initialize } + override def associatedFile(self: Symbol): AbstractFile = { + val sourceModule = self.sourceModule + sourceModule.rawInfo match { + case loader: SymbolLoader => loader.associatedFile(sourceModule) + case _ => super.associatedFile(self) + } + } } /** used from classfile parser to avoid cycles */ diff --git a/src/compiler/scala/tools/nsc/symtab/classfile/AbstractFileReader.scala b/src/compiler/scala/tools/nsc/symtab/classfile/AbstractFileReader.scala index a8d673663e8d..19be00dd686a 100644 --- a/src/compiler/scala/tools/nsc/symtab/classfile/AbstractFileReader.scala +++ b/src/compiler/scala/tools/nsc/symtab/classfile/AbstractFileReader.scala @@ -25,11 +25,8 @@ import scala.tools.nsc.io.AbstractFile * @author Philippe Altherr * @version 1.0, 23/03/2004 */ -class AbstractFileReader(val file: AbstractFile) { - - /** the buffer containing the file - */ - val buf: Array[Byte] = file.toByteArray +class AbstractFileReader(val file: AbstractFile, val buf: Array[Byte]) { + def this(file: AbstractFile) = this(file, file.toByteArray) /** the current input pointer */ diff --git a/src/compiler/scala/tools/nsc/symtab/classfile/ClassfileParser.scala b/src/compiler/scala/tools/nsc/symtab/classfile/ClassfileParser.scala index 81f8dfe44543..fcb1b247f07b 100644 --- a/src/compiler/scala/tools/nsc/symtab/classfile/ClassfileParser.scala +++ b/src/compiler/scala/tools/nsc/symtab/classfile/ClassfileParser.scala @@ -15,18 +15,17 @@ package tools.nsc package symtab package classfile -import java.io.{ByteArrayInputStream, DataInputStream, File, IOException} +import java.io._ import java.lang.Integer.toHexString -import scala.collection.{immutable, mutable} -import scala.collection.mutable.{ArrayBuffer, ListBuffer} import scala.annotation.switch -import scala.reflect.internal.JavaAccFlags +import scala.collection.mutable.{ArrayBuffer, ListBuffer} +import scala.collection.{immutable, mutable} +import scala.reflect.internal.{JavaAccFlags, SymbolTable} import scala.reflect.internal.pickling.{ByteCodecs, PickleBuffer} import scala.reflect.io.NoAbstractFile -import scala.reflect.internal.util.Collections._ -import scala.tools.nsc.util.ClassPath import scala.tools.nsc.io.AbstractFile +import scala.tools.nsc.util.ClassPath import scala.util.control.NonFatal /** This abstract class implements a class file parser. @@ -57,8 +56,9 @@ abstract class ClassfileParser { def classPath: ClassPath import definitions._ - import scala.reflect.internal.ClassfileConstants._ + import Flags._ + import scala.reflect.internal.ClassfileConstants._ protected type ThisConstantPool <: ConstantPool protected def newConstantPool: ThisConstantPool @@ -152,14 +152,21 @@ abstract class ClassfileParser { def parse(file: AbstractFile, clazz: ClassSymbol, module: ModuleSymbol): Unit = { this.file = file pushBusy(clazz) { - this.in = new AbstractFileReader(file) this.clazz = clazz this.staticModule = module this.isScala = false - parseHeader() - this.pool = newConstantPool - parseClass() + this.in = new AbstractFileReader(file) + val magic = in.getInt(in.bp) + if (magic != JAVA_MAGIC && file.name.endsWith(".sig")) { + currentClass = TermName(clazz.javaClassName) + isScala = true + unpickler.unpickle(in.buf, 0, clazz, staticModule, file.name) + } else { + parseHeader() + this.pool = newConstantPool + parseClass() + } } } @@ -441,6 +448,15 @@ abstract class ClassfileParser { lookupClass(name) } + // TODO: remove after the next 2.13 milestone + // A bug in the backend caused classes ending in `$` do get only a Scala marker attribute + // instead of a ScalaSig and a Signature annotaiton. This went unnoticed because isScalaRaw + // classes were parsed like Java classes. The below covers the cases in the std lib. + private def isNothingOrNull = { + val n = clazz.fullName.toString + n == "scala.runtime.Nothing$" || n == "scala.runtime.Null$" + } + def parseClass() { val jflags = readClassFlags() val sflags = jflags.toScalaFlags @@ -890,7 +906,6 @@ abstract class ClassfileParser { case Some(san: AnnotationInfo) => val bytes = san.assocs.find({ _._1 == nme.bytes }).get._2.asInstanceOf[ScalaSigBytes].bytes - unpickler.unpickle(bytes, 0, clazz, staticModule, in.file.name) case None => throw new RuntimeException("Scala class file does not contain Scala annotation") diff --git a/src/compiler/scala/tools/nsc/symtab/classfile/Pickler.scala b/src/compiler/scala/tools/nsc/symtab/classfile/Pickler.scala index 947b95f57baa..fa0bb189892a 100644 --- a/src/compiler/scala/tools/nsc/symtab/classfile/Pickler.scala +++ b/src/compiler/scala/tools/nsc/symtab/classfile/Pickler.scala @@ -90,6 +90,8 @@ abstract class Pickler extends SubComponent { throw e } } + + override protected def shouldSkipThisPhaseForJava: Boolean = false //from some -Y ?? } private class Pickle(root: Symbol) extends PickleBuffer(new Array[Byte](4096), -1, 0) { @@ -213,7 +215,7 @@ abstract class Pickler extends SubComponent { // initially, but seems not to work, as the bug shows). // Adding the LOCAL_CHILD is necessary to retain exhaustivity warnings under separate // compilation. See test neg/aladdin1055. - val parents = (if (sym.isTrait) List(definitions.ObjectTpe) else Nil) ::: List(sym.tpe) + val parents = if (sym.isTrait) List(definitions.ObjectTpe, sym.tpe) else List(sym.tpe) globals + sym.newClassWithInfo(tpnme.LOCAL_CHILD, parents, EmptyScope, pos = sym.pos) } diff --git a/src/compiler/scala/tools/nsc/typechecker/Analyzer.scala b/src/compiler/scala/tools/nsc/typechecker/Analyzer.scala index b068e43d1ad4..bc5ffd0ccd7c 100644 --- a/src/compiler/scala/tools/nsc/typechecker/Analyzer.scala +++ b/src/compiler/scala/tools/nsc/typechecker/Analyzer.scala @@ -112,11 +112,13 @@ trait Analyzer extends AnyRef try { val typer = newTyper(rootContext(unit)) unit.body = typer.typed(unit.body) - for (workItem <- unit.toCheck) workItem() - if (settings.warnUnusedImport) - warnUnusedImports(unit) - if (settings.warnUnused.isSetByUser) - new checkUnused(typer).apply(unit) + if (!settings.Youtline.value) { + for (workItem <- unit.toCheck) workItem() + if (settings.warnUnusedImport) + warnUnusedImports(unit) + if (settings.warnUnused.isSetByUser) + new checkUnused(typer).apply(unit) + } } finally { unit.toCheck.clear() diff --git a/src/compiler/scala/tools/nsc/typechecker/Contexts.scala b/src/compiler/scala/tools/nsc/typechecker/Contexts.scala index ad643bc9a9f8..59d2d06356b3 100644 --- a/src/compiler/scala/tools/nsc/typechecker/Contexts.scala +++ b/src/compiler/scala/tools/nsc/typechecker/Contexts.scala @@ -1072,14 +1072,25 @@ trait Contexts { self: Analyzer => || unit.exists && s.sourceFile != unit.source.file ) ) - def lookupInPrefix(name: Name) = { + def lookupInPrefix(name: Name): Symbol = { val sym = pre.member(name).filter(qualifies) def isNonPackageNoModuleClass(sym: Symbol) = sym.isClass && !sym.isModuleClass && !sym.isPackageClass if (!sym.exists && unit.isJava && isNonPackageNoModuleClass(pre.typeSymbol)) { + var baseClasses = pre.baseClasses + while (baseClasses != Nil) { + val base = baseClasses.head + val pre1 = companionSymbolOf(base, this).typeOfThis + val sym = pre1.member(name).filter(qualifies) + .andAlso(_ => pre = pre1) + if (sym != NoSymbol) { + pre = pre1 + return sym + } + baseClasses = baseClasses.tail + } + NoSymbol // TODO factor out duplication with Typer::inCompanionForJavaStatic - val pre1 = companionSymbolOf(pre.typeSymbol, this).typeOfThis - pre1.member(name).filter(qualifies).andAlso(_ => pre = pre1) } else sym } def accessibleInPrefix(s: Symbol) = isAccessible(s, pre, superAccess = false) diff --git a/src/compiler/scala/tools/nsc/typechecker/Implicits.scala b/src/compiler/scala/tools/nsc/typechecker/Implicits.scala index 891691370527..2cc7fa729899 100644 --- a/src/compiler/scala/tools/nsc/typechecker/Implicits.scala +++ b/src/compiler/scala/tools/nsc/typechecker/Implicits.scala @@ -87,6 +87,15 @@ trait Implicits { * @return A search result */ def inferImplicit(tree: Tree, pt: Type, reportAmbiguous: Boolean, isView: Boolean, context: Context, saveAmbiguousDivergent: Boolean, pos: Position): SearchResult = { + currentRun.profiler.beforeImplicitSearch(pt) + try { + inferImplicit1(tree, pt, reportAmbiguous, isView, context, saveAmbiguousDivergent, pos) + } finally { + currentRun.profiler.afterImplicitSearch(pt) + } + } + + private def inferImplicit1(tree: Tree, pt: Type, reportAmbiguous: Boolean, isView: Boolean, context: Context, saveAmbiguousDivergent: Boolean, pos: Position): SearchResult = { // Note that the isInvalidConversionTarget seems to make a lot more sense right here, before all the // work is performed, than at the point where it presently exists. val shouldPrint = printTypings && !context.undetparams.isEmpty diff --git a/src/compiler/scala/tools/nsc/typechecker/Macros.scala b/src/compiler/scala/tools/nsc/typechecker/Macros.scala index e837e0eb8270..85b2293d1286 100644 --- a/src/compiler/scala/tools/nsc/typechecker/Macros.scala +++ b/src/compiler/scala/tools/nsc/typechecker/Macros.scala @@ -14,6 +14,8 @@ package scala.tools.nsc package typechecker import java.lang.Math.min +import java.net.URL + import symtab.Flags._ import scala.reflect.internal.util.ScalaClassLoader import scala.reflect.runtime.ReflectionUtils @@ -22,6 +24,7 @@ import scala.reflect.internal.TypesStats import scala.reflect.macros.util._ import scala.util.control.ControlThrowable import scala.reflect.internal.util.ListOfNil +import scala.reflect.io.AbstractFile import scala.reflect.macros.runtime.{AbortMacroException, MacroRuntimes} import scala.reflect.macros.compiler.DefaultMacroCompiler import scala.tools.reflect.FastTrack @@ -64,49 +67,6 @@ trait Macros extends MacroRuntimes with Traces with Helpers { def globalSettings = global.settings - private final val macroClassLoadersCache = - new scala.tools.nsc.classpath.FileBasedCache[ScalaClassLoader]() - - /** Obtains a `ClassLoader` instance used for macro expansion. - * - * By default a new `ScalaClassLoader` is created using the classpath - * from global and the classloader of self as parent. - * - * Mirrors with runtime definitions (e.g. Repl) need to adjust this method. - */ - protected def findMacroClassLoader(): ClassLoader = { - val classpath = global.classPath.asURLs - def newLoader = () => { - macroLogVerbose("macro classloader: initializing from -cp: %s".format(classpath)) - ScalaClassLoader.fromURLs(classpath, self.getClass.getClassLoader) - } - - val disableCache = settings.YcacheMacroClassLoader.value == settings.CachePolicy.None.name - if (disableCache) newLoader() - else { - import scala.tools.nsc.io.Jar - import scala.reflect.io.{AbstractFile, Path} - - val urlsAndFiles = classpath.map(u => u -> AbstractFile.getURL(u)) - val hasNullURL = urlsAndFiles.filter(_._2 eq null) - if (hasNullURL.nonEmpty) { - // TODO if the only null is jrt:// we can still cache - // TODO filter out classpath elements pointing to non-existing files before we get here, that's another source of null - macroLogVerbose(s"macro classloader: caching is disabled because `AbstractFile.getURL` returned `null` for ${hasNullURL.map(_._1).mkString(", ")}.") - newLoader() - } else { - val locations = urlsAndFiles.map(t => Path(t._2.file)) - val nonJarZips = locations.filterNot(Jar.isJarOrZip(_)) - if (nonJarZips.nonEmpty) { - macroLogVerbose(s"macro classloader: caching is disabled because the following paths are not supported: ${nonJarZips.mkString(",")}.") - newLoader() - } else { - macroClassLoadersCache.getOrCreate(locations.map(_.jfile.toPath()), newLoader) - } - } - } - } - /** `MacroImplBinding` and its companion module are responsible for * serialization/deserialization of macro def -> impl bindings. * @@ -770,7 +730,13 @@ trait Macros extends MacroRuntimes with Traces with Helpers { // By default, use the current typer's fresh name creator in macros. The compiler option // allows people to opt in to the old behaviour of Scala 2.12, which used a global fresh creator. if (!settings.YmacroFresh.value) currentFreshNameCreator = typer.fresh - pluginsMacroExpand(typer, expandee, mode, pt) + val macroSym = expandee.symbol + currentRun.profiler.beforeMacroExpansion(macroSym) + try { + pluginsMacroExpand(typer, expandee, mode, pt) + } finally { + currentRun.profiler.afterMacroExpansion(macroSym) + } } /** Default implementation of `macroExpand`. @@ -968,6 +934,11 @@ trait Macros extends MacroRuntimes with Traces with Helpers { }.transform(expandee) } +object Macros { + final val macroClassLoadersCache = + new scala.tools.nsc.classpath.FileBasedCache[ScalaClassLoader.URLClassLoader]() +} + trait MacrosStats { self: TypesStats with Statistics => val macroExpandCount = newCounter ("#macro expansions", "typer") diff --git a/src/compiler/scala/tools/nsc/typechecker/MethodSynthesis.scala b/src/compiler/scala/tools/nsc/typechecker/MethodSynthesis.scala index 20535e89f413..898fce90cef3 100644 --- a/src/compiler/scala/tools/nsc/typechecker/MethodSynthesis.scala +++ b/src/compiler/scala/tools/nsc/typechecker/MethodSynthesis.scala @@ -231,6 +231,13 @@ trait MethodSynthesis { val methDef = factoryMeth(classDef.mods & AccessFlags | METHOD | IMPLICIT | SYNTHETIC, classDef.name.toTermName, classDef) val methSym = enterInScope(assignMemberSymbol(methDef)) context.unit.synthetics(methSym) = methDef + + treeInfo.firstConstructor(classDef.impl.body) match { + case primaryConstructor: DefDef => + if (mexists(primaryConstructor.vparamss)(_.mods.hasDefault)) + enterDefaultGetters(methSym, primaryConstructor, primaryConstructor.vparamss, primaryConstructor.tparams) + case _ => + } methSym setInfo implicitFactoryMethodCompleter(methDef, classDef.symbol) } diff --git a/src/compiler/scala/tools/nsc/typechecker/Typers.scala b/src/compiler/scala/tools/nsc/typechecker/Typers.scala index acac49cff07d..6f1ac036364b 100644 --- a/src/compiler/scala/tools/nsc/typechecker/Typers.scala +++ b/src/compiler/scala/tools/nsc/typechecker/Typers.scala @@ -1834,38 +1834,43 @@ trait Typers extends Adaptations with Tags with TypersTracking with PatternTyper def typedClassDef(cdef: ClassDef): Tree = { val clazz = cdef.symbol - val typedMods = typedModifiers(cdef.mods) - assert(clazz != NoSymbol, cdef) - reenterTypeParams(cdef.tparams) - val tparams1 = cdef.tparams mapConserve (typedTypeDef) - val impl1 = newTyper(context.make(cdef.impl, clazz, newScope)).typedTemplate(cdef.impl, typedParentTypes(cdef.impl)) - val impl2 = finishMethodSynthesis(impl1, clazz, context) - if (clazz.isTrait && clazz.info.parents.nonEmpty && clazz.info.firstParent.typeSymbol == AnyClass) - checkEphemeral(clazz, impl2.body) - - if ((clazz isNonBottomSubClass ClassfileAnnotationClass) && (clazz != ClassfileAnnotationClass)) { - if (!clazz.owner.isPackageClass) - context.error(clazz.pos, "inner classes cannot be classfile annotations") - // Ignore @SerialVersionUID, because it is special-cased and handled completely differently. - // It only extends ClassfileAnnotationClass instead of StaticAnnotation to get the enforcement - // of constant argument values "for free". Related to scala/bug#7041. - else if (clazz != SerialVersionUIDAttr) restrictionWarning(cdef.pos, unit, - """|subclassing Classfile does not - |make your annotation visible at runtime. If that is what - |you want, you must write the annotation class in Java.""".stripMargin) - } - - warnTypeParameterShadow(tparams1, clazz) - - if (!isPastTyper) { - for (ann <- clazz.getAnnotation(DeprecatedAttr)) { - val m = companionSymbolOf(clazz, context) - if (m != NoSymbol) - m.moduleClass.addAnnotation(AnnotationInfo(ann.atp, ann.args, List())) - } - } - treeCopy.ClassDef(cdef, typedMods, cdef.name, tparams1, impl2) - .setType(NoType) + currentRun.profiler.beforeTypedImplDef(clazz) + try { + val typedMods = typedModifiers(cdef.mods) + assert(clazz != NoSymbol, cdef) + reenterTypeParams(cdef.tparams) + val tparams1 = cdef.tparams mapConserve (typedTypeDef) + val impl1 = newTyper(context.make(cdef.impl, clazz, newScope)).typedTemplate(cdef.impl, typedParentTypes(cdef.impl)) + val impl2 = finishMethodSynthesis(impl1, clazz, context) + if (clazz.isTrait && clazz.info.parents.nonEmpty && clazz.info.firstParent.typeSymbol == AnyClass) + checkEphemeral(clazz, impl2.body) + + if ((clazz isNonBottomSubClass ClassfileAnnotationClass) && (clazz != ClassfileAnnotationClass)) { + if (!clazz.owner.isPackageClass) + context.error(clazz.pos, "inner classes cannot be classfile annotations") + // Ignore @SerialVersionUID, because it is special-cased and handled completely differently. + // It only extends ClassfileAnnotationClass instead of StaticAnnotation to get the enforcement + // of constant argument values "for free". Related to scala/bug#7041. + else if (clazz != SerialVersionUIDAttr) restrictionWarning(cdef.pos, unit, + """|subclassing Classfile does not + |make your annotation visible at runtime. If that is what + |you want, you must write the annotation class in Java.""".stripMargin) + } + + warnTypeParameterShadow(tparams1, clazz) + + if (!isPastTyper) { + for (ann <- clazz.getAnnotation(DeprecatedAttr)) { + val m = companionSymbolOf(clazz, context) + if (m != NoSymbol) + m.moduleClass.addAnnotation(AnnotationInfo(ann.atp, ann.args, List())) + } + } + treeCopy.ClassDef(cdef, typedMods, cdef.name, tparams1, impl2) + .setType(NoType) + } finally { + currentRun.profiler.afterTypedImplDef(clazz) + } } def typedModuleDef(mdef: ModuleDef): Tree = { @@ -1875,31 +1880,37 @@ trait Typers extends Adaptations with Tags with TypersTracking with PatternTyper if (linkedClass != NoSymbol) linkedClass.info.decl(nme.CONSTRUCTOR).alternatives foreach (_.initialize) - val clazz = mdef.symbol.moduleClass - val typedMods = typedModifiers(mdef.mods) - assert(clazz != NoSymbol, mdef) - val noSerializable = ( - (linkedClass eq NoSymbol) - || linkedClass.isErroneous - || !linkedClass.isSerializable - || clazz.isSerializable - ) - val impl1 = newTyper(context.make(mdef.impl, clazz, newScope)).typedTemplate(mdef.impl, { - typedParentTypes(mdef.impl) ++ ( - if (noSerializable) Nil - else { - clazz.makeSerializable() - TypeTree(SerializableTpe).setPos(clazz.pos.focus) :: Nil - } - ) - }) + val clazz = mdef.symbol.moduleClass + currentRun.profiler.beforeTypedImplDef(clazz) + try { - val impl2 = finishMethodSynthesis(impl1, clazz, context) + val typedMods = typedModifiers(mdef.mods) + assert(clazz != NoSymbol, mdef) + val noSerializable = ( + (linkedClass eq NoSymbol) + || linkedClass.isErroneous + || !linkedClass.isSerializable + || clazz.isSerializable + ) + val impl1 = newTyper(context.make(mdef.impl, clazz, newScope)).typedTemplate(mdef.impl, { + typedParentTypes(mdef.impl) ++ ( + if (noSerializable) Nil + else { + clazz.makeSerializable() + TypeTree(SerializableTpe).setPos(clazz.pos.focus) :: Nil + } + ) + }) + + val impl2 = finishMethodSynthesis(impl1, clazz, context) - if (settings.isScala211 && mdef.symbol == PredefModule) - ensurePredefParentsAreInSameSourceFile(impl2) + if (settings.isScala211 && mdef.symbol == PredefModule) + ensurePredefParentsAreInSameSourceFile(impl2) - treeCopy.ModuleDef(mdef, typedMods, mdef.name, impl2) setType NoType + treeCopy.ModuleDef(mdef, typedMods, mdef.name, impl2) setType NoType + } finally { + currentRun.profiler.afterTypedImplDef(clazz) + } } private def ensurePredefParentsAreInSameSourceFile(template: Template) = { @@ -2047,18 +2058,24 @@ trait Typers extends Adaptations with Tags with TypersTracking with PatternTyper def typedValDef(vdef: ValDef): ValDef = { val sym = vdef.symbol - val valDefTyper = { - val maybeConstrCtx = - if ((sym.isParameter || sym.isEarlyInitialized) && sym.owner.isConstructor) context.makeConstructorContext - else context - newTyper(maybeConstrCtx.makeNewScope(vdef, sym)) + currentRun.profiler.beforeTypedImplDef(sym) + try { + val valDefTyper = { + val maybeConstrCtx = + if ((sym.isParameter || sym.isEarlyInitialized) && sym.owner.isConstructor) context.makeConstructorContext + else context + newTyper(maybeConstrCtx.makeNewScope(vdef, sym)) + } + valDefTyper.typedValDefImpl(vdef) + } finally { + currentRun.profiler.afterTypedImplDef(sym) } - valDefTyper.typedValDefImpl(vdef) } // use typedValDef instead. this version is called after creating a new context for the ValDef - private def typedValDefImpl(vdef: ValDef) = { + private def typedValDefImpl(vdef: ValDef): ValDef = { val sym = vdef.symbol.initialize + val typedMods = if (nme.isLocalName(sym.name) && sym.isPrivateThis && !vdef.mods.isPrivateLocal) { // scala/bug#10009 This tree has been given a field symbol by `enterGetterSetter`, patch up the // modifiers accordingly so that we can survive resetAttrs and retypechecking. @@ -2268,89 +2285,92 @@ trait Typers extends Adaptations with Tags with TypersTracking with PatternTyper } def typedDefDef(ddef: DefDef): DefDef = { - // an accessor's type completer may mutate a type inside `ddef` (`== context.unit.synthetics(ddef.symbol)`) - // concretely: it sets the setter's parameter type or the getter's return type (when derived from a valdef with empty tpt) val meth = ddef.symbol.initialize + currentRun.profiler.beforeTypedImplDef(meth) + try { - reenterTypeParams(ddef.tparams) - reenterValueParams(ddef.vparamss) + reenterTypeParams(ddef.tparams) + reenterValueParams(ddef.vparamss) - // for `val` and `var` parameter, look at `target` meta-annotation - if (!isPastTyper && meth.isPrimaryConstructor) { - for (vparams <- ddef.vparamss; vd <- vparams) { - if (vd.mods.isParamAccessor) { - vd.symbol setAnnotations (vd.symbol.annotations filter AnnotationInfo.mkFilter(ParamTargetClass, defaultRetention = true)) + // for `val` and `var` parameter, look at `target` meta-annotation + if (!isPastTyper && meth.isPrimaryConstructor) { + for (vparams <- ddef.vparamss; vd <- vparams) { + if (vd.mods.isParamAccessor) { + vd.symbol setAnnotations (vd.symbol.annotations filter AnnotationInfo.mkFilter(ParamTargetClass, defaultRetention = true)) + } } } - } - val tparams1 = ddef.tparams mapConserve typedTypeDef - val vparamss1 = ddef.vparamss mapConserve (_ mapConserve typedValDef) + val tparams1 = ddef.tparams mapConserve typedTypeDef + val vparamss1 = ddef.vparamss mapConserve (_ mapConserve typedValDef) - warnTypeParameterShadow(tparams1, meth) + warnTypeParameterShadow(tparams1, meth) - meth.annotations.map(_.completeInfo()) + meth.annotations.map(_.completeInfo()) - for (vparams1 <- vparamss1; vparam1 <- vparams1 dropRight 1) - if (isRepeatedParamType(vparam1.symbol.tpe)) - StarParamNotLastError(vparam1) + for (vparams1 <- vparamss1; vparam1 <- vparams1 dropRight 1) + if (isRepeatedParamType(vparam1.symbol.tpe)) + StarParamNotLastError(vparam1) - val tpt1 = checkNoEscaping.privates(this, meth, typedType(ddef.tpt)) - checkNonCyclic(ddef, tpt1) - ddef.tpt.setType(tpt1.tpe) - val typedMods = typedModifiers(ddef.mods) - var rhs1 = - if (ddef.name == nme.CONSTRUCTOR && !ddef.symbol.hasStaticFlag) { // need this to make it possible to generate static ctors - if (!meth.isPrimaryConstructor && + val tpt1 = checkNoEscaping.privates(this, meth, typedType(ddef.tpt)) + checkNonCyclic(ddef, tpt1) + ddef.tpt.setType(tpt1.tpe) + val typedMods = typedModifiers(ddef.mods) + var rhs1 = + if (ddef.name == nme.CONSTRUCTOR && !ddef.symbol.hasStaticFlag) { // need this to make it possible to generate static ctors + if (!meth.isPrimaryConstructor && (!meth.owner.isClass || - meth.owner.isModuleClass || - meth.owner.isAnonOrRefinementClass)) - InvalidConstructorDefError(ddef) - typed(ddef.rhs) - } else if (meth.isMacro) { - // typechecking macro bodies is sort of unconventional - // that's why we employ our custom typing scheme orchestrated outside of the typer - transformedOr(ddef.rhs, typedMacroBody(this, ddef)) - } else { - transformedOrTyped(ddef.rhs, EXPRmode, tpt1.tpe) - } + meth.owner.isModuleClass || + meth.owner.isAnonOrRefinementClass)) + InvalidConstructorDefError(ddef) + typed(ddef.rhs) + } else if (meth.isMacro) { + // typechecking macro bodies is sort of unconventional + // that's why we employ our custom typing scheme orchestrated outside of the typer + transformedOr(ddef.rhs, typedMacroBody(this, ddef)) + } else { + transformedOrTyped(ddef.rhs, EXPRmode, tpt1.tpe) + } - if (meth.isClassConstructor && !isPastTyper && !meth.owner.isSubClass(AnyValClass) && !meth.isJava) { - // There are no supercalls for AnyVal or constructors from Java sources, which + if (meth.isClassConstructor && !isPastTyper && !meth.owner.isSubClass(AnyValClass) && !meth.isJava) { + // There are no supercalls for AnyVal or constructors from Java sources, which // would blow up in analyzeSuperConsructor; there's nothing to be computed for them - // anyway. - if (meth.isPrimaryConstructor) + // anyway. + if (meth.isPrimaryConstructor) analyzeSuperConsructor(meth, vparamss1, rhs1) - else - checkSelfConstructorArgs(ddef, meth.owner) - } + else + checkSelfConstructorArgs(ddef, meth.owner) + } - if (tpt1.tpe.typeSymbol != NothingClass && !context.returnsSeen && rhs1.tpe.typeSymbol != NothingClass) - rhs1 = checkDead(context, rhs1) + if (tpt1.tpe.typeSymbol != NothingClass && !context.returnsSeen && rhs1.tpe.typeSymbol != NothingClass) + rhs1 = checkDead(context, rhs1) - if (!isPastTyper && meth.owner.isClass && + if (!isPastTyper && meth.owner.isClass && meth.paramss.exists(ps => ps.exists(_.hasDefault) && isRepeatedParamType(ps.last.tpe))) - StarWithDefaultError(meth) - - if (!isPastTyper) { - val allParams = meth.paramss.flatten - for (p <- allParams) { - for (n <- p.deprecatedParamName) { - if (allParams.exists(p1 => p != p1 && (p1.name == n || p1.deprecatedParamName.exists(_ == n)))) - DeprecatedParamNameError(p, n) + StarWithDefaultError(meth) + + if (!isPastTyper) { + val allParams = meth.paramss.flatten + for (p <- allParams) { + for (n <- p.deprecatedParamName) { + if (allParams.exists(p1 => p != p1 && (p1.name == n || p1.deprecatedParamName.exists(_ == n)))) + DeprecatedParamNameError(p, n) + } } - } - if (meth.isStructuralRefinementMember) - checkMethodStructuralCompatible(ddef) + if (meth.isStructuralRefinementMember) + checkMethodStructuralCompatible(ddef) - if (meth.isImplicit && !meth.isSynthetic) meth.info.paramss match { - case List(param) :: _ if !param.isImplicit => - checkFeature(ddef.pos, currentRun.runDefinitions.ImplicitConversionsFeature, meth.toString) - case _ => + if (meth.isImplicit && !meth.isSynthetic) meth.info.paramss match { + case List(param) :: _ if !param.isImplicit => + checkFeature(ddef.pos, currentRun.runDefinitions.ImplicitConversionsFeature, meth.toString) + case _ => + } } - } - treeCopy.DefDef(ddef, typedMods, ddef.name, tparams1, vparamss1, tpt1, rhs1) setType NoType + treeCopy.DefDef(ddef, typedMods, ddef.name, tparams1, vparamss1, tpt1, rhs1) setType NoType + } finally { + currentRun.profiler.afterTypedImplDef(meth) + } } def typedTypeDef(tdef: TypeDef): TypeDef = @@ -5845,7 +5865,7 @@ trait Typers extends Adaptations with Tags with TypersTracking with PatternTyper final def transformedOrTyped(tree: Tree, mode: Mode, pt: Type): Tree = { lookupTransformed(tree) match { case Some(tree1) => tree1 - case _ => typed(tree, mode, pt) + case _ => if (settings.Youtline.value) EmptyTree else typed(tree, mode, pt) } } final def lookupTransformed(tree: Tree): Option[Tree] = diff --git a/src/compiler/scala/tools/nsc/util/ClassPath.scala b/src/compiler/scala/tools/nsc/util/ClassPath.scala index 827c7ce5dbd7..b2ad49c5a38c 100644 --- a/src/compiler/scala/tools/nsc/util/ClassPath.scala +++ b/src/compiler/scala/tools/nsc/util/ClassPath.scala @@ -13,6 +13,8 @@ package scala.tools.nsc package util +import java.io.Closeable + import io.{AbstractFile, Directory, File, Jar} import java.net.MalformedURLException import java.net.URL diff --git a/src/compiler/scala/tools/reflect/ReflectGlobal.scala b/src/compiler/scala/tools/reflect/ReflectGlobal.scala index e1cf834c6fb6..2efd699e9f44 100644 --- a/src/compiler/scala/tools/reflect/ReflectGlobal.scala +++ b/src/compiler/scala/tools/reflect/ReflectGlobal.scala @@ -25,18 +25,14 @@ import scala.tools.nsc.typechecker.Analyzer class ReflectGlobal(currentSettings: Settings, reporter: Reporter, override val rootClassLoader: ClassLoader) extends Global(currentSettings, reporter) with scala.tools.reflect.ReflectSetup with scala.reflect.runtime.SymbolTable { - override lazy val analyzer = new { - val global: ReflectGlobal.this.type = ReflectGlobal.this - } with Analyzer { - /** Obtains the classLoader used for runtime macro expansion. - * - * Macro expansion can use everything available in [[global.classPath]] or [[rootClassLoader]]. - * The [[rootClassLoader]] is used to obtain runtime defined macros. - */ - override protected def findMacroClassLoader(): ClassLoader = { - val classpath = global.classPath.asURLs - ScalaClassLoader.fromURLs(classpath, rootClassLoader) - } + /** Obtains the classLoader used for runtime macro expansion. + * + * Macro expansion can use everything available in `global.classPath` or `rootClassLoader`. + * The `rootClassLoader` is used to obtain runtime defined macros. + */ + override protected[scala] def findMacroClassLoader(): ClassLoader = { + val classpath = classPath.asURLs + perRunCaches.recordClassloader(ScalaClassLoader.fromURLs(classpath, rootClassLoader)) } override def transformedType(sym: Symbol) = diff --git a/src/compiler/scala/tools/reflect/ReflectMain.scala b/src/compiler/scala/tools/reflect/ReflectMain.scala index 3abd5f390761..a290c6bfafc8 100644 --- a/src/compiler/scala/tools/reflect/ReflectMain.scala +++ b/src/compiler/scala/tools/reflect/ReflectMain.scala @@ -14,15 +14,13 @@ package scala.tools package reflect import scala.reflect.internal.util.ScalaClassLoader -import scala.tools.nsc.Driver -import scala.tools.nsc.Global -import scala.tools.nsc.Settings +import scala.tools.nsc.{Driver, Global, CloseableRegistry, Settings} import scala.tools.util.PathResolver object ReflectMain extends Driver { private def classloaderFromSettings(settings: Settings) = { - val classPathURLs = new PathResolver(settings).resultAsURLs + val classPathURLs = new PathResolver(settings, new CloseableRegistry).resultAsURLs ScalaClassLoader.fromURLs(classPathURLs, getClass.getClassLoader) } diff --git a/src/compiler/scala/tools/util/PathResolver.scala b/src/compiler/scala/tools/util/PathResolver.scala index 1ad471e40f8b..cf454d5854f8 100644 --- a/src/compiler/scala/tools/util/PathResolver.scala +++ b/src/compiler/scala/tools/util/PathResolver.scala @@ -15,8 +15,9 @@ package tools package util import java.net.URL + import scala.tools.reflect.WrappedProperties.AccessControl -import scala.tools.nsc.Settings +import scala.tools.nsc.{CloseableRegistry, Settings} import scala.tools.nsc.util.ClassPath import scala.reflect.io.{Directory, File, Path} import PartialFunction.condOpt @@ -189,19 +190,24 @@ object PathResolver { } else { val settings = new Settings() val rest = settings.processArguments(args.toList, processAll = false)._2 - val pr = new PathResolver(settings) - println("COMMAND: 'scala %s'".format(args.mkString(" "))) - println("RESIDUAL: 'scala %s'\n".format(rest.mkString(" "))) - - pr.result match { - case cp: AggregateClassPath => - println(s"ClassPath has ${cp.aggregates.size} entries and results in:\n${cp.asClassPathStrings}") + val registry = new CloseableRegistry + try { + val pr = new PathResolver(settings, registry) + println("COMMAND: 'scala %s'".format(args.mkString(" "))) + println("RESIDUAL: 'scala %s'\n".format(rest.mkString(" "))) + + pr.result match { + case cp: AggregateClassPath => + println(s"ClassPath has ${cp.aggregates.size} entries and results in:\n${cp.asClassPathStrings}") + } + } finally { + registry.close() } } } -final class PathResolver(settings: Settings) { - private val classPathFactory = new ClassPathFactory(settings) +final class PathResolver(settings: Settings, closeableRegistry: CloseableRegistry) { + private val classPathFactory = new ClassPathFactory(settings, closeableRegistry) import PathResolver.{ AsLines, Defaults, ppcp } @@ -250,7 +256,7 @@ final class PathResolver(settings: Settings) { // Assemble the elements! def basis = List[Traversable[ClassPath]]( - JrtClassPath.apply(settings.releaseValue), // 0. The Java 9 classpath (backed by the jrt:/ virtual system, if available) + jrt, // 0. The Java 9+ classpath (backed by the ct.sym or jrt:/ virtual system, if available) classesInPath(javaBootClassPath), // 1. The Java bootstrap class path. contentsOfDirsInPath(javaExtDirs), // 2. The Java extension class path. classesInExpandedPath(javaUserClassPath), // 3. The Java application class path. @@ -261,6 +267,8 @@ final class PathResolver(settings: Settings) { sourcesInPath(sourcePath) // 7. The Scala source path. ) + private def jrt: Option[ClassPath] = JrtClassPath.apply(settings.releaseValue, closeableRegistry) + lazy val containers = basis.flatten.distinct override def toString = s""" diff --git a/src/partest-extras/scala/tools/partest/BytecodeTest.scala b/src/partest-extras/scala/tools/partest/BytecodeTest.scala index b016778bf428..309a6d49c482 100644 --- a/src/partest-extras/scala/tools/partest/BytecodeTest.scala +++ b/src/partest-extras/scala/tools/partest/BytecodeTest.scala @@ -18,6 +18,7 @@ import scala.tools.asm.tree._ import java.io.{InputStream, File => JFile} import AsmNode._ +import scala.tools.nsc.CloseableRegistry /** * Provides utilities for inspecting bytecode using ASM library. @@ -144,7 +145,7 @@ abstract class BytecodeTest { import scala.tools.nsc.Settings // logic inspired by scala.tools.util.PathResolver implementation // `Settings` is used to check YdisableFlatCpCaching in ZipArchiveFlatClassPath - val factory = new ClassPathFactory(new Settings()) + val factory = new ClassPathFactory(new Settings(), new CloseableRegistry) val containers = factory.classesInExpandedPath(sys.props("partest.output") + java.io.File.pathSeparator + Defaults.javaUserClassPath) new AggregateClassPath(containers) } diff --git a/src/reflect/scala/reflect/internal/Definitions.scala b/src/reflect/scala/reflect/internal/Definitions.scala index 05aebaf3ca1e..c233fd4ffbd3 100644 --- a/src/reflect/scala/reflect/internal/Definitions.scala +++ b/src/reflect/scala/reflect/internal/Definitions.scala @@ -1236,7 +1236,7 @@ trait Definitions extends api.StandardDefinitions { lazy val AnnotationDefaultAttr: ClassSymbol = { val sym = RuntimePackageClass.newClassSymbol(tpnme.AnnotationDefaultATTR, NoPosition, 0L) - sym setInfo ClassInfoType(List(AnnotationClass.tpe), newScope, sym) + sym setInfo ClassInfoType(List(StaticAnnotationClass.tpe), newScope, sym) markAllCompleted(sym) RuntimePackageClass.info.decls.toList.filter(_.name == sym.name) match { case existing :: _ => diff --git a/src/reflect/scala/reflect/internal/SymbolTable.scala b/src/reflect/scala/reflect/internal/SymbolTable.scala index 99fd5edd7ac7..20593d875c69 100644 --- a/src/reflect/scala/reflect/internal/SymbolTable.scala +++ b/src/reflect/scala/reflect/internal/SymbolTable.scala @@ -14,6 +14,8 @@ package scala package reflect package internal +import java.net.URLClassLoader + import scala.annotation.elidable import scala.collection.mutable import util._ @@ -21,6 +23,7 @@ import java.util.concurrent.TimeUnit import scala.collection.mutable.ArrayBuffer import scala.reflect.internal.{TreeGen => InternalTreeGen} +import scala.reflect.io.AbstractFile abstract class SymbolTable extends macros.Universe with Collections @@ -416,6 +419,22 @@ abstract class SymbolTable extends macros.Universe cache } + /** Closes the provided classloader at the conclusion of this Run */ + final def recordClassloader(loader: ClassLoader): ClassLoader = { + def attemptClose(loader: ClassLoader): Unit = { + loader match { + case u: URLClassLoader => debuglog("Closing classloader " + u); u.close() + case _ => + } + } + caches ::= new WeakReference((new Clearable { + def clear(): Unit = { + attemptClose(loader) + } + })) + loader + } + /** * Removes a cache from the per-run caches. This is useful for testing: it allows running the * compiler and then inspect the state of a cache. @@ -493,6 +512,9 @@ abstract class SymbolTable extends macros.Universe * Adds the `sm` String interpolator to a [[scala.StringContext]]. */ implicit val StringContextStripMarginOps: StringContext => StringContextStripMarginOps = util.StringContextStripMarginOps + + protected[scala] def currentRunProfilerBeforeCompletion(root: Symbol, associatedFile: AbstractFile): Unit = () + protected[scala] def currentRunProfilerAfterCompletion(root: Symbol, associatedFile: AbstractFile): Unit = () } trait SymbolTableStats { diff --git a/src/reflect/scala/reflect/internal/Symbols.scala b/src/reflect/scala/reflect/internal/Symbols.scala index 3a25d830a20f..b8c54bfe2934 100644 --- a/src/reflect/scala/reflect/internal/Symbols.scala +++ b/src/reflect/scala/reflect/internal/Symbols.scala @@ -1978,7 +1978,7 @@ trait Symbols extends api.Symbols { self: SymbolTable => var alts0: List[Symbol] = alternatives var alts1: List[Symbol] = Nil - while (alts0.nonEmpty) { + while (!alts0.isEmpty) { if (cond(alts0.head)) alts1 ::= alts0.head else diff --git a/src/reflect/scala/reflect/internal/pickling/UnPickler.scala b/src/reflect/scala/reflect/internal/pickling/UnPickler.scala index 7b82aa3e9f24..c1fc858cef11 100644 --- a/src/reflect/scala/reflect/internal/pickling/UnPickler.scala +++ b/src/reflect/scala/reflect/internal/pickling/UnPickler.scala @@ -249,6 +249,9 @@ abstract class UnPickler { else NoSymbol } + if (owner == definitions.ScalaPackageClass && name == tpnme.AnyRef) + return definitions.AnyRefClass + // (1) Try name. localDummy orElse fromName(name) orElse { // (2) Try with expanded name. Can happen if references to private diff --git a/src/reflect/scala/reflect/internal/util/ChromeTrace.scala b/src/reflect/scala/reflect/internal/util/ChromeTrace.scala new file mode 100644 index 000000000000..69da5d5982c9 --- /dev/null +++ b/src/reflect/scala/reflect/internal/util/ChromeTrace.scala @@ -0,0 +1,189 @@ +/* + * Scala (https://www.scala-lang.org) + * + * Copyright EPFL and Lightbend, Inc. + * + * Licensed under Apache License 2.0 + * (http://www.apache.org/licenses/LICENSE-2.0). + * + * See the NOTICE file distributed with this work for + * additional information regarding copyright ownership. + */ + +package scala.reflect.internal.util + +import java.io.Closeable +import java.lang.management.ManagementFactory +import java.nio.file.{Files, Path} +import java.util +import java.util.concurrent.TimeUnit + +import scala.collection.mutable + +object ChromeTrace { + + private object EventType { + final val Start = "B" + final val Instant = "I" + final val End = "E" + final val Complete = "X" + + final val Counter = "C" + + final val AsyncStart = "b" + final val AsyncInstant = "n" + final val AsyncEnd = "e" + } + +} + +/** Allows writing a subset of of https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview# + * for use in Chrome's about://tracing or the tooling in https://www.google.com.au/search?q=catapult+tracing&oq=catapult+tracing+&aqs=chrome..69i57.3974j0j4&sourceid=chrome&ie=UTF-8 */ +final class ChromeTrace(f: Path) extends Closeable { + import ChromeTrace.EventType + private val traceWriter = FileUtils.newAsyncBufferedWriter(f) + private val context = mutable.ArrayStack[JsonContext](TopContext) + private val tidCache = new ThreadLocal[String]() { + override def initialValue(): String = Thread.currentThread().getId.formatted("%05d") + } + objStart() + fld("traceEvents") + context.push(ValueContext) + arrStart() + traceWriter.newLine() + + private val pid = ManagementFactory.getRuntimeMXBean().getName().replaceAll("@.*", "") + + override def close(): Unit = { + arrEnd() + objEnd() + context.pop() + tidCache.remove() + traceWriter.close() + } + + def traceDurationEvent(name: String, startNanos: Long, durationNanos: Long, tid: String = this.tid(), pidSuffix: String = ""): Unit = { + val durationMicros = nanosToMicros(durationNanos) + val startMicros = nanosToMicros(startNanos) + objStart() + str("cat", "scalac") + str("name", name) + str("ph", EventType.Complete) + str("tid", tid) + writePid(pidSuffix) + lng("ts", startMicros) + lng("dur", durationMicros) + objEnd() + traceWriter.newLine() + } + + private def writePid(pidSuffix: String) = { + if (pidSuffix == "") + str("pid", pid) + else + str2("pid", pid, "-", pidSuffix) + } + + def traceCounterEvent(name: String, counterName: String, count: Long, processWide: Boolean): Unit = { + objStart() + str("cat", "scalac") + str("name", name) + str("ph", EventType.Counter) + str("tid", tid()) + writePid(pidSuffix = if (processWide) "" else tid()) + lng("ts", microTime()) + fld("args") + objStart() + lng(counterName, count) + objEnd() + objEnd() + traceWriter.newLine() + } + + def traceDurationEventStart(cat: String, name: String, colour: String = "", pidSuffix: String = tid()): Unit = traceDurationEventStartEnd(EventType.Start, cat, name, colour, pidSuffix) + def traceDurationEventEnd(cat: String, name: String, colour: String = "", pidSuffix: String = tid()): Unit = traceDurationEventStartEnd(EventType.End, cat, name, colour, pidSuffix) + + private def traceDurationEventStartEnd(eventType: String, cat: String, name: String, colour: String, pidSuffix: String = ""): Unit = { + objStart() + str("cat", cat) + str("name", name) + str("ph", eventType) + writePid(pidSuffix) + str("tid", tid()) + lng("ts", microTime()) + if (colour != "") { + str("cname", colour) + } + objEnd() + traceWriter.newLine() + } + + private def tid(): String = tidCache.get() + + private def nanosToMicros(t: Long): Long = TimeUnit.NANOSECONDS.toMicros(t) + + private def microTime(): Long = nanosToMicros(System.nanoTime()) + + sealed abstract class JsonContext + case class ArrayContext(var first: Boolean) extends JsonContext + case class ObjectContext(var first: Boolean) extends JsonContext + case object ValueContext extends JsonContext + case object TopContext extends JsonContext + + private def str(name: String, value: String): Unit = { + fld(name) + traceWriter.write("\"") + traceWriter.write(value) // This assumes no escaping is needed + traceWriter.write("\"") + } + private def str2(name: String, value: String, valueContinued1: String, valueContinued2: String): Unit = { + fld(name) + traceWriter.write("\"") + traceWriter.write(value) // This assumes no escaping is needed + traceWriter.write(valueContinued1) // This assumes no escaping is needed + traceWriter.write(valueContinued2) // This assumes no escaping is needed + traceWriter.write("\"") + } + private def lng(name: String, value: Long): Unit = { + fld(name) + traceWriter.write(String.valueOf(value)) + traceWriter.write("") + } + private def objStart(): Unit = { + context.top match { + case ac @ ArrayContext(first) => + if (first) ac.first = false + else traceWriter.write(",") + case _ => + } + context.push(ObjectContext(true)) + traceWriter.write("{") + } + private def objEnd(): Unit = { + traceWriter.write("}") + context.pop() + } + private def arrStart(): Unit = { + traceWriter.write("[") + context.push(ArrayContext(true)) + } + private def arrEnd(): Unit = { + traceWriter.write("]") + context.pop() + } + + private def fld(name: String) = { + val topContext = context.top + topContext match { + case oc @ ObjectContext(first) => + if (first) oc.first = false + else traceWriter.write(",") + case context => + throw new IllegalStateException("Wrong context: " + context) + } + traceWriter.write("\"") + traceWriter.write(name) + traceWriter.write("\"") + traceWriter.write(":") + } +} diff --git a/src/reflect/scala/reflect/internal/util/FileUtils.scala b/src/reflect/scala/reflect/internal/util/FileUtils.scala new file mode 100644 index 000000000000..ef5955775648 --- /dev/null +++ b/src/reflect/scala/reflect/internal/util/FileUtils.scala @@ -0,0 +1,199 @@ +/* + * Scala (https://www.scala-lang.org) + * + * Copyright EPFL and Lightbend, Inc. + * + * Licensed under Apache License 2.0 + * (http://www.apache.org/licenses/LICENSE-2.0). + * + * See the NOTICE file distributed with this work for + * additional information regarding copyright ownership. + */ + +package scala.reflect.internal.util + +import java.io.{BufferedWriter, IOException, OutputStreamWriter, Writer} +import java.nio.CharBuffer +import java.nio.charset.{Charset, CharsetEncoder, StandardCharsets} +import java.nio.file.{Files, OpenOption, Path} +import java.util.concurrent.LinkedBlockingQueue +import java.util.concurrent.atomic.AtomicBoolean + + +import scala.concurrent.duration.Duration +import scala.concurrent.{Await, Promise} +import scala.util.{Failure, Success} + +object FileUtils { + def newAsyncBufferedWriter(path: Path, charset: Charset = StandardCharsets.UTF_8, options: Array[OpenOption] = NO_OPTIONS, threadsafe: Boolean = false): LineWriter = { + val encoder: CharsetEncoder = charset.newEncoder + val writer = new OutputStreamWriter(Files.newOutputStream(path, options: _*), encoder) + newAsyncBufferedWriter(new BufferedWriter(writer), threadsafe) + } + def newAsyncBufferedWriter(underlying: Writer, threadsafe: Boolean): LineWriter = { + val async = new AsyncBufferedWriter(underlying) + if (threadsafe) new ThreadsafeWriter(async) else async + } + private val NO_OPTIONS = new Array[OpenOption](0) + + sealed abstract class LineWriter extends Writer { + def newLine(): Unit + } + private class ThreadsafeWriter(val underlying: AsyncBufferedWriter) extends LineWriter { + lock = underlying + override def write(c: Int): Unit = + lock.synchronized (underlying.write(c)) + + override def write(cbuf: Array[Char]): Unit = + lock.synchronized (underlying.write(cbuf)) + + override def write(cbuf: Array[Char], off: Int, len: Int): Unit = + lock.synchronized (underlying.write(cbuf, off, len)) + + override def write(str: String): Unit = + lock.synchronized (underlying.write(str)) + + override def write(str: String, off: Int, len: Int): Unit = + lock.synchronized (underlying.write(str, off, len)) + + override def flush(): Unit = + lock.synchronized (underlying.flush()) + + override def close(): Unit = + lock.synchronized (underlying.close()) + + override def newLine(): Unit = + lock.synchronized (underlying.newLine()) + + } + + private object AsyncBufferedWriter { + private val Close = CharBuffer.allocate(0) + private val Flush = CharBuffer.allocate(0) + } + private class AsyncBufferedWriter(val underlying: Writer, bufferSize : Int = 4096) extends LineWriter { + private var current: CharBuffer = allocate + override def write(c: Int): Unit = super.write(c) + private def flushAsync(): Unit = { + background.ensureProcessed(current) + current = allocate + } +// allocate or reuse a CharArray which is guaranteed to have a backing array + private def allocate: CharBuffer = { + val reused = background.reuseBuffer + if (reused eq null) CharBuffer.allocate(bufferSize) + else { + //we don't care about race conditions + background.reuseBuffer = null + reused.clear() + reused + } + } + + override def write(cbuf: Array[Char], initialOffset: Int, initialLength: Int): Unit = { + var offset = initialOffset + var length = initialLength + while (length > 0) { + val capacity = current.remaining() + if (length <= capacity) { + current.put(cbuf, offset, length) + length = 0 + } else { + current.put(cbuf, offset, capacity) + flushAsync() + length -= capacity + offset += capacity + } + } + } + + override def write(s: String, initialOffset: Int, initialLength: Int): Unit = { + var offset = initialOffset + var length = initialLength + while (length > 0) { + val capacity = current.remaining() + if (length <= capacity) { + current.put(s, offset, offset + length) + length = 0 + } else { + current.put(s, offset, offset + capacity) + flushAsync() + length -= capacity + offset += capacity + } + } + } + + def newLine(): Unit = write(scala.util.Properties.lineSeparator) + + /** slightly breaks the flush contract in that the flush is not complete when the method returns */ + override def flush(): Unit = { + flushAsync() + } + + override def close(): Unit = { + background.ensureProcessed(current) + background.ensureProcessed(AsyncBufferedWriter.Close) + current = null + Await.result(background.asyncStatus.future, Duration.Inf) + underlying.close() + } + private object background extends Runnable{ + + import scala.concurrent.ExecutionContext.Implicits.global + + private val pending = new LinkedBlockingQueue[CharBuffer] + //a failure detected will case an Failure, Success indicates a close + val asyncStatus = Promise[Unit]() + private val scheduled = new AtomicBoolean + @volatile var reuseBuffer: CharBuffer = _ + + def ensureProcessed(buffer: CharBuffer): Unit = { + if (asyncStatus.isCompleted) { + asyncStatus.future.value.get match { + case Success(()) => throw new IllegalStateException("closed") + case Failure(t) => throw new IOException("async failure", t) + } + } + + //order is essential - add to the queue before the CAS + pending.add(buffer) + if (scheduled.compareAndSet(false, true)) { + global.execute(background) + } + } + + def run(): Unit = { + try { + while (!pending.isEmpty) { + val next = pending.poll() + if (next eq AsyncBufferedWriter.Flush) { + underlying.flush() + } else if (next eq AsyncBufferedWriter.Close) { + underlying.flush() + underlying.close() + asyncStatus.trySuccess(()) + } else { + val array = next.array() + next.flip() + underlying.write(array, next.arrayOffset() + next.position(), next.limit()) + reuseBuffer = next + } + } + } catch { + case t: Throwable => + asyncStatus.tryFailure(t) + throw t + } + finally scheduled.set(false) + + //we are not scheduled any more + //as a last check ensure that we didnt race with an addition to the queue + //order is essential - queue is checked before CAS + if ((!pending.isEmpty) && scheduled.compareAndSet(false, true)) { + global.execute(background) + } + } + } + } +} diff --git a/src/reflect/scala/reflect/io/RootPath.scala b/src/reflect/scala/reflect/io/RootPath.scala new file mode 100644 index 000000000000..51273a9c3f37 --- /dev/null +++ b/src/reflect/scala/reflect/io/RootPath.scala @@ -0,0 +1,39 @@ +package scala.reflect.io + +import java.io.Closeable +import java.nio +import java.nio.file.{FileSystems, Files} + + +abstract class RootPath extends Closeable { + def root: nio.file.Path +} + +object RootPath { + def apply(path: nio.file.Path, writable: Boolean): RootPath = { + if (path.getFileName.toString.endsWith(".jar")) { + import java.net.URI + val zipFile = URI.create("jar:file:" + path.toUri.getPath) + val env = new java.util.HashMap[String, String]() + if (!Files.exists(path.getParent)) + Files.createDirectories(path.getParent) + if (writable) { + env.put("create", "true") + if (Files.exists(path)) + Files.delete(path) + } + val zipfs = FileSystems.newFileSystem(zipFile, env) + new RootPath { + def root = zipfs.getRootDirectories.iterator().next() + def close(): Unit = { + zipfs.close() + } + } + } else { + new RootPath { + override def root: nio.file.Path = path + override def close(): Unit = () + } + } + } +} \ No newline at end of file diff --git a/src/reflect/scala/reflect/io/ZipArchive.scala b/src/reflect/scala/reflect/io/ZipArchive.scala index ee109799f3d5..05c591b9d535 100644 --- a/src/reflect/scala/reflect/io/ZipArchive.scala +++ b/src/reflect/scala/reflect/io/ZipArchive.scala @@ -149,6 +149,7 @@ abstract class ZipArchive(override val file: JFile, release: Option[String]) ext if (entry.isDirectory) ensureDir(dirs, entry.getName, entry) else ensureDir(dirs, dirName(entry.getName), null) } + def close(): Unit } /** ''Note: This library is considered experimental and should not be used unless you know what you are doing.'' */ final class FileZipArchive(file: JFile, release: Option[String]) extends ZipArchive(file, release) { @@ -232,6 +233,7 @@ final class FileZipArchive(file: JFile, release: Option[String]) extends ZipArch } } finally { if (ZipArchive.closeZipFile) zipFile.close() + else closeables ::= zipFile } root } @@ -259,6 +261,10 @@ final class FileZipArchive(file: JFile, release: Option[String]) extends ZipArch case x: FileZipArchive => file.getAbsoluteFile == x.file.getAbsoluteFile case _ => false } + private[this] var closeables: List[java.io.Closeable] = Nil + override def close(): Unit = { + closeables.foreach(_.close) + } } /** ''Note: This library is considered experimental and should not be used unless you know what you are doing.'' */ final class URLZipArchive(val url: URL) extends ZipArchive(null) { @@ -266,6 +272,7 @@ final class URLZipArchive(val url: URL) extends ZipArchive(null) { val root = new DirEntry("/") val dirs = mutable.HashMap[String, DirEntry]("" -> root) val in = new ZipInputStream(new ByteArrayInputStream(Streamable.bytes(input))) + closeables ::= in @tailrec def loop() { val zipEntry = in.getNextEntry() @@ -327,6 +334,10 @@ final class URLZipArchive(val url: URL) extends ZipArchive(null) { case x: URLZipArchive => url == x.url case _ => false } + private[this] var closeables: List[java.io.Closeable] = Nil + def close(): Unit = { + closeables.foreach(_.close()) + } } final class ManifestResources(val url: URL) extends ZipArchive(null) { @@ -334,6 +345,8 @@ final class ManifestResources(val url: URL) extends ZipArchive(null) { val root = new DirEntry("/") val dirs = mutable.HashMap[String, DirEntry]("" -> root) val manifest = new Manifest(input) + closeables ::= input + val iter = manifest.getEntries().keySet().iterator().asScala.filter(_.endsWith(".class")).map(new ZipEntry(_)) for (zipEntry <- iter) { @@ -385,4 +398,8 @@ final class ManifestResources(val url: URL) extends ZipArchive(null) { } } } + private[this] var closeables: List[java.io.Closeable] = Nil + override def close(): Unit = { + closeables.foreach(_.close()) + } } diff --git a/src/repl/scala/tools/nsc/interpreter/ILoop.scala b/src/repl/scala/tools/nsc/interpreter/ILoop.scala index a32e2aa02ee7..7e4ac310c4d4 100644 --- a/src/repl/scala/tools/nsc/interpreter/ILoop.scala +++ b/src/repl/scala/tools/nsc/interpreter/ILoop.scala @@ -363,7 +363,8 @@ class ILoop(in0: Option[BufferedReader], protected val out: JPrintWriter) extend } private def changeSettings(line: String): Result = { - def showSettings() = for (s <- settings.userSetSettings.toSeq.sorted) echo(s.toString) + val settings1 = settings + def showSettings() = for (s <- settings1.userSetSettings.toSeq.sorted) echo(s.toString) if (line.isEmpty) showSettings() else { updateSettings(line) ; () } } private def updateSettings(line: String) = { diff --git a/src/repl/scala/tools/nsc/interpreter/IMain.scala b/src/repl/scala/tools/nsc/interpreter/IMain.scala index 764bb4d48543..73cbc828eea4 100644 --- a/src/repl/scala/tools/nsc/interpreter/IMain.scala +++ b/src/repl/scala/tools/nsc/interpreter/IMain.scala @@ -28,6 +28,7 @@ import scala.tools.nsc.util._ import ScalaClassLoader.URLClassLoader import scala.tools.nsc.util.Exceptional.unwrap import java.net.URL +import java.io.Closeable import scala.tools.util.PathResolver import scala.util.{Try => Trying} @@ -63,7 +64,7 @@ import scala.util.{Try => Trying} * @author Moez A. Abdel-Gawad * @author Lex Spoon */ -class IMain(initialSettings: Settings, protected val out: JPrintWriter) extends Imports with PresentationCompilation { +class IMain(initialSettings: Settings, protected val out: JPrintWriter) extends Imports with PresentationCompilation with Closeable { imain => def this(initialSettings: Settings) = this(initialSettings, IMain.defaultOut) @@ -100,7 +101,7 @@ class IMain(initialSettings: Settings, protected val out: JPrintWriter) extends def compilerClasspath: Seq[java.net.URL] = ( if (isInitializeComplete) global.classPath.asURLs - else new PathResolver(settings).resultAsURLs // the compiler's classpath + else new PathResolver(settings, global.closeableRegistry).resultAsURLs // the compiler's classpath ) def settings = initialSettings // Run the code body with the given boolean settings flipped to true. @@ -683,6 +684,9 @@ class IMain(initialSettings: Settings, protected val out: JPrintWriter) extends */ def close() { reporter.flush() + if (isInitializeComplete) { + global.close() + } } /** Here is where we: diff --git a/src/repl/scala/tools/nsc/interpreter/PresentationCompilation.scala b/src/repl/scala/tools/nsc/interpreter/PresentationCompilation.scala index 4c7f05318c53..296cc5a4b1df 100644 --- a/src/repl/scala/tools/nsc/interpreter/PresentationCompilation.scala +++ b/src/repl/scala/tools/nsc/interpreter/PresentationCompilation.scala @@ -16,7 +16,7 @@ import scala.reflect.internal.util.RangePosition import scala.reflect.io.AbstractFile import scala.tools.nsc.backend.JavaPlatform import scala.tools.nsc.util.ClassPath -import scala.tools.nsc.{interactive, Settings} +import scala.tools.nsc.{interactive, CloseableRegistry, Settings} import scala.tools.nsc.reporters.StoreReporter import scala.tools.nsc.classpath._ @@ -63,10 +63,6 @@ trait PresentationCompilation { * You may downcast the `reporter` to `StoreReporter` to access type errors. */ def newPresentationCompiler(): interactive.Global = { - def mergedFlatClasspath = { - val replOutClasspath = ClassPathFactory.newClassPath(replOutput.dir, settings) - AggregateClassPath(replOutClasspath :: global.platform.classPath :: Nil) - } def copySettings: Settings = { val s = new Settings(_ => () /* ignores "bad option -nc" errors, etc */) s.processArguments(global.settings.recreateArgs, processAll = false) @@ -75,6 +71,11 @@ trait PresentationCompilation { } val storeReporter: StoreReporter = new StoreReporter val interactiveGlobal = new interactive.Global(copySettings, storeReporter) { self => + def mergedFlatClasspath = { + val replOutClasspath = ClassPathFactory.newClassPath(replOutput.dir, settings, closeableRegistry) + AggregateClassPath(replOutClasspath :: global.platform.classPath :: Nil) + } + override lazy val platform: ThisPlatform = { new JavaPlatform { lazy val global: self.type = self diff --git a/src/repl/scala/tools/nsc/interpreter/ReplGlobal.scala b/src/repl/scala/tools/nsc/interpreter/ReplGlobal.scala index 1273d6ac92fb..72b5a7424ceb 100644 --- a/src/repl/scala/tools/nsc/interpreter/ReplGlobal.scala +++ b/src/repl/scala/tools/nsc/interpreter/ReplGlobal.scala @@ -30,16 +30,11 @@ trait ReplGlobal extends Global { super.abort(msg) } - override lazy val analyzer = new { - val global: ReplGlobal.this.type = ReplGlobal.this - } with Analyzer { - - override protected def findMacroClassLoader(): ClassLoader = { - val loader = super.findMacroClassLoader - macroLogVerbose("macro classloader: initializing from a REPL classloader: %s".format(global.classPath.asURLs)) - val virtualDirectory = globalSettings.outputDirs.getSingleOutput.get - new util.AbstractFileClassLoader(virtualDirectory, loader) {} - } + override protected[scala] def findMacroClassLoader(): ClassLoader = { + val loader = super.findMacroClassLoader + analyzer.macroLogVerbose("macro classloader: initializing from a REPL classloader: %s".format(classPath.asURLs)) + val virtualDirectory = analyzer.globalSettings.outputDirs.getSingleOutput.get + new util.AbstractFileClassLoader(virtualDirectory, loader) {} } override def optimizerClassPath(base: ClassPath): ClassPath = { @@ -47,7 +42,7 @@ trait ReplGlobal extends Global { case None => base case Some(out) => // Make bytecode of previous lines available to the inliner - val replOutClasspath = ClassPathFactory.newClassPath(settings.outputDirs.getSingleOutput.get, settings) + val replOutClasspath = ClassPathFactory.newClassPath(settings.outputDirs.getSingleOutput.get, settings, closeableRegistry) AggregateClassPath.createAggregate(platform.classPath, replOutClasspath) } } diff --git a/src/scalap/scala/tools/scalap/Main.scala b/src/scalap/scala/tools/scalap/Main.scala index 42b0fdfb236a..5e3d633d429f 100644 --- a/src/scalap/scala/tools/scalap/Main.scala +++ b/src/scalap/scala/tools/scalap/Main.scala @@ -14,8 +14,9 @@ package scala package tools.scalap import java.io.{ByteArrayOutputStream, OutputStreamWriter, PrintStream} + import scala.reflect.NameTransformer -import scala.tools.nsc.Settings +import scala.tools.nsc.{CloseableRegistry, Settings} import scala.tools.nsc.classpath.{AggregateClassPath, ClassPathFactory} import scala.tools.nsc.util.ClassPath import scala.tools.util.PathResolver @@ -185,14 +186,18 @@ object Main extends Main { settings.YdisableFlatCpCaching.value = arguments contains opts.disableFlatClassPathCaching settings.Ylogcp.value = arguments contains opts.logClassPath - val path = createClassPath(cpArg, settings) - - // print the classpath if output is verbose - if (verbose) - Console.println(Console.BOLD + "CLASSPATH" + Console.RESET + " = " + path.asClassPathString) - - // process all given classes - arguments.getOthers foreach process(arguments, path) + val registry = new CloseableRegistry + try { + val path = createClassPath(cpArg, settings, registry) + // print the classpath if output is verbose + if (verbose) + Console.println(Console.BOLD + "CLASSPATH" + Console.RESET + " = " + path.asClassPathString) + + // process all given classes + arguments.getOthers foreach process(arguments, path) + } finally { + registry.close() + } } private def parseArguments(args: Array[String]) = @@ -208,11 +213,11 @@ object Main extends Main { .withOption(opts.logClassPath) .parse(args) - private def createClassPath(cpArg: Option[String], settings: Settings) = cpArg match { + private def createClassPath(cpArg: Option[String], settings: Settings, closeableRegistry: CloseableRegistry) = cpArg match { case Some(cp) => - AggregateClassPath(new ClassPathFactory(settings).classesInExpandedPath(cp)) + AggregateClassPath(new ClassPathFactory(settings, closeableRegistry).classesInExpandedPath(cp)) case _ => settings.classpath.value = "." // include '.' in the default classpath scala/bug#6669 - new PathResolver(settings).result + new PathResolver(settings, closeableRegistry).result } } diff --git a/test/files/jvm/strictfp/StrictFpJava.java b/test/files/jvm/strictfp/StrictFpJava.java new file mode 100644 index 000000000000..89e4e94ae528 --- /dev/null +++ b/test/files/jvm/strictfp/StrictFpJava.java @@ -0,0 +1,5 @@ +strictfp class StrictFpJava {} + +class StrictFpJavaMethod { + strictfp void test() {} +} diff --git a/test/files/pos/java-ref-super-class-static/J.java b/test/files/pos/java-ref-super-class-static/J.java new file mode 100644 index 000000000000..0bd7545e6e9c --- /dev/null +++ b/test/files/pos/java-ref-super-class-static/J.java @@ -0,0 +1,12 @@ +class J { + static class Base { + static class StaticInner {} + class Inner {} + } + static class Sub extends Base { + void f1( Inner inner) {} + + void f2( StaticInner inner) {} // not found: "StaticInner" + void f3(Base.StaticInner inner) {} // workaround: qualifiy + } +} diff --git a/test/files/presentation/parse-invariants.check b/test/files/presentation/parse-invariants.check index 32e9c846ab5c..961bc6df7938 100644 --- a/test/files/presentation/parse-invariants.check +++ b/test/files/presentation/parse-invariants.check @@ -1,3 +1,10 @@ +parseTree +NoNewSymbolsEntered OK +Unique OK +Unattributed OK +NeverModify OK +AlwaysParseTree OK +parseTree NoNewSymbolsEntered OK Unique OK Unattributed OK diff --git a/test/files/presentation/parse-invariants/Test.scala b/test/files/presentation/parse-invariants/Test.scala index 128896ccaae0..c16045ce912e 100644 --- a/test/files/presentation/parse-invariants/Test.scala +++ b/test/files/presentation/parse-invariants/Test.scala @@ -5,12 +5,16 @@ import scala.tools.nsc.interactive.Response object Test extends InteractiveTest { override def execute(): Unit = { - val sf = sourceFiles.find(_.file.name == "A.scala").head - noNewSymbols(sf) - uniqueParseTree(sf) - unattributedParseTree(sf) - neverModifyParseTree(sf) - shouldAlwaysReturnParseTree(sf) + def test(fileName: String): Unit = { + val sf = sourceFiles.find(_.file.name == fileName).head + noNewSymbols(sf) + uniqueParseTree(sf) + unattributedParseTree(sf) + neverModifyParseTree(sf) + shouldAlwaysReturnParseTree(sf) + } + test("A.scala") + test("A.java") } /** @@ -19,12 +23,13 @@ object Test extends InteractiveTest { private def noNewSymbols(sf: SourceFile) { def nextId() = compiler.NoSymbol.newTermSymbol(compiler.TermName("dummy"), compiler.NoPosition, compiler.NoFlags).id val id = nextId() + println("parseTree") val tree = compiler.parseTree(sf) val id2 = nextId() if (id2 == id + 1) { reporter.println("NoNewSymbolsEntered OK") } else { - reporter.println("NoNewSymbolsEntered FAILED") + reporter.println("NoNewSymbolsEntered FAILED. ") } } diff --git a/test/files/presentation/parse-invariants/src/a/A.java b/test/files/presentation/parse-invariants/src/a/A.java new file mode 100644 index 000000000000..a04478149103 --- /dev/null +++ b/test/files/presentation/parse-invariants/src/a/A.java @@ -0,0 +1,16 @@ +package syntax; + +class A { + transient volatile int x; + strictfp void test() { + } + + native void nativeMethod() + + synchronized void syncMethod() {} + + void thrower() throws Throwable {} + +} + +strictfp class B {} \ No newline at end of file diff --git a/test/files/run/implicit-class-implicit-param-with-default.check b/test/files/run/implicit-class-implicit-param-with-default.check new file mode 100644 index 000000000000..f0ab6fd76b86 --- /dev/null +++ b/test/files/run/implicit-class-implicit-param-with-default.check @@ -0,0 +1,5 @@ +default +default +default +explicit +explicit diff --git a/test/files/run/implicit-class-implicit-param-with-default.scala b/test/files/run/implicit-class-implicit-param-with-default.scala new file mode 100644 index 000000000000..41bebbc8960a --- /dev/null +++ b/test/files/run/implicit-class-implicit-param-with-default.scala @@ -0,0 +1,17 @@ +object Test { + implicit class C(self: String)(implicit val foo: String = "default") { + def test = foo + } + + implicit class WorkaroundOk(self: String)(implicit val foo: String) { + def this(self: String, dummy: AnyRef = null) { this(self)("")} + } + + def main(args: Array[String]) { + println("".foo) + println(C("").foo) + println(new C("").foo) + println(C("")("explicit").foo) + println(new C("")("explicit").foo) + } +} diff --git a/test/junit/scala/reflect/internal/util/FileUtilsTest.scala b/test/junit/scala/reflect/internal/util/FileUtilsTest.scala new file mode 100644 index 000000000000..21eba42985b9 --- /dev/null +++ b/test/junit/scala/reflect/internal/util/FileUtilsTest.scala @@ -0,0 +1,89 @@ +package scala.reflect.internal.util + +import java.io._ + +import org.junit.Assert._ +import org.junit._ + +class FileUtilsTest { + + @Test def writeIsSame(): Unit = { + val fileTest = File.createTempFile("FileUtilsTest", "t1") + val fileExpected = File.createTempFile("FileUtilsTest", "t2") + + val sTest = FileUtils.newAsyncBufferedWriter(new FileWriter(fileTest), false) + val sExpected = new BufferedWriter(new FileWriter(fileExpected)) + + def writeBoth(s:String, asChars: Boolean) = { + if (asChars) { + sTest.write(s.toCharArray) + sExpected.write(s.toCharArray) + } else { + sTest.write(s) + sExpected.write(s) + } + } + + for (i <- 1 to 2000) { + writeBoth(s"line $i text;", true) + writeBoth(s"line $i chars", false) + sTest.newLine + sExpected.newLine + } + sTest.close() + sExpected.close() + + assertEquals(fileExpected.length(),fileTest.length()) + + val expIn = new BufferedReader(new FileReader(fileExpected)) + val testIn = new BufferedReader(new FileReader(fileTest)) + + var exp = expIn.readLine() + while (exp ne null) { + val actual = testIn.readLine() + assertEquals(exp, actual) + exp = expIn.readLine() + } + expIn.close() + testIn.close() + fileTest.delete() + fileExpected.delete() + } + + @Test def showPerformance: Unit = { + //warmup + for (i <- 1 to 1000) { + writeIsSame() + } + + val fileTest = File.createTempFile("FileUtilsTest", "t1") + val fileExpected = File.createTempFile("FileUtilsTest", "t2") + + for (i <- 1 to 10) { + val sTest = FileUtils.newAsyncBufferedWriter(fileTest.toPath) + val sExpected = new BufferedWriter(new FileWriter(fileExpected)) + + val t1 = System.nanoTime() + List.tabulate(10000) {i => + sTest.write(s"line $i text;") + sTest.newLine + } + val t2 = System.nanoTime() + sTest.close() + val t3 = System.nanoTime() + List.tabulate(10000) {i => + sExpected.write(s"line $i text;") + sExpected.newLine + } + val t4 = System.nanoTime() + sExpected.close() + + println(s"async took ${t2 - t1} ns") + println(s"buffered took ${t4 - t3} ns") + + fileTest.delete() + fileExpected.delete() + } + } + +} diff --git a/test/junit/scala/tools/nsc/GlobalCustomizeClassloaderTest.scala b/test/junit/scala/tools/nsc/GlobalCustomizeClassloaderTest.scala new file mode 100644 index 000000000000..500379706090 --- /dev/null +++ b/test/junit/scala/tools/nsc/GlobalCustomizeClassloaderTest.scala @@ -0,0 +1,72 @@ +package scala.tools.nsc + +import org.junit.{Assert, Test} +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 + +import scala.reflect.internal.util.{AbstractFileClassLoader, NoSourceFile} +import scala.reflect.io.{Path, VirtualDirectory} +import scala.tools.nsc.plugins.{Plugin, PluginComponent} + +@RunWith(classOf[JUnit4]) +class GlobalCustomizeClassloaderTest { + // Demonstrate extension points to customise creation of the classloaders used to load compiler + // plugins and macro implementations. + // + // A use case could be for a build tool to take control of caching of these classloaders in a way + // that properly closes them before one of the elements needs to be overwritten. + @Test def test(): Unit = { + val g = new Global(new Settings) { + override protected[scala] def findMacroClassLoader(): ClassLoader = getClass.getClassLoader + override protected def findPluginClassLoader(classpath: Seq[Path]): ClassLoader = { + val d = new VirtualDirectory("", None) + val xml = d.fileNamed("scalac-plugin.xml") + val out = xml.bufferedOutput + out.write( + s""" + |sample-plugin + |${classOf[SamplePlugin].getName} + | + |""".stripMargin.getBytes()) + out.close() + new AbstractFileClassLoader(d, getClass.getClassLoader) + } + } + g.settings.usejavacp.value = true + g.settings.plugin.value = List("sample") + new g.Run + assert(g.settings.log.value == List("typer")) + + val unit = new g.CompilationUnit(NoSourceFile) + val context = g.analyzer.rootContext(unit) + val typer = g.analyzer.newTyper(context) + import g._ + SampleMacro.data = "in this classloader" + val typed = typer.typed(q"scala.tools.nsc.SampleMacro.m") + assert(!reporter.hasErrors) + typed match { + case Typed(Literal(Constant(s: String)), _) => Assert.assertEquals(SampleMacro.data, s) + case _ => Assert.fail() + } + g.close() + } +} + +object SampleMacro { + var data: String = _ + import language.experimental.macros + import scala.reflect.macros.blackbox.Context + def m: String = macro impl + def impl(c: Context): c.Tree = c.universe.Literal(c.universe.Constant(data)) +} + +class SamplePlugin(val global: Global) extends Plugin { + override val name: String = "sample" + override val description: String = "sample" + override val components: List[PluginComponent] = Nil + override def init(options: List[String], error: String => Unit): Boolean = { + val result = super.init(options, error) + global.settings.log.value = List("typer") + result + } +} diff --git a/test/junit/scala/tools/nsc/classpath/JrtClassPathTest.scala b/test/junit/scala/tools/nsc/classpath/JrtClassPathTest.scala index b46677d6d477..fdc2b9caae69 100644 --- a/test/junit/scala/tools/nsc/classpath/JrtClassPathTest.scala +++ b/test/junit/scala/tools/nsc/classpath/JrtClassPathTest.scala @@ -8,7 +8,7 @@ import org.junit.Test import org.junit.runner.RunWith import org.junit.runners.JUnit4 -import scala.tools.nsc.Settings +import scala.tools.nsc.{CloseableRegistry, Settings} import scala.tools.nsc.backend.jvm.AsmUtils import scala.tools.nsc.util.ClassPath import scala.tools.util.PathResolver @@ -19,14 +19,15 @@ class JrtClassPathTest { @Test def lookupJavaClasses(): Unit = { val specVersion = scala.util.Properties.javaSpecVersion // Run the test using the JDK8 or 9 provider for rt.jar depending on the platform the test is running on. + val closeableRegistry = new CloseableRegistry val cp: ClassPath = if (specVersion == "" || specVersion == "1.8") { val settings = new Settings() - val resolver = new PathResolver(settings) - val elements = new ClassPathFactory(settings).classesInPath(resolver.Calculated.javaBootClassPath) + val resolver = new PathResolver(settings, closeableRegistry) + val elements = new ClassPathFactory(settings, closeableRegistry).classesInPath(resolver.Calculated.javaBootClassPath) AggregateClassPath(elements) } - else JrtClassPath(None).get + else JrtClassPath(None, closeableRegistry).get assertEquals(Nil, cp.classes("")) assertTrue(cp.packages("java").toString, cp.packages("java").exists(_.name == "java.lang")) @@ -37,5 +38,7 @@ class JrtClassPathTest { assertTrue(cp.list("java.lang").classesAndSources.exists(_.name == "Object")) assertTrue(cp.findClass("java.lang.Object").isDefined) assertTrue(cp.findClassFile("java.lang.Object").isDefined) + + closeableRegistry.close() } } diff --git a/test/junit/scala/tools/nsc/classpath/PathResolverBaseTest.scala b/test/junit/scala/tools/nsc/classpath/PathResolverBaseTest.scala index d3d4289d8b94..e8025ec69ec5 100644 --- a/test/junit/scala/tools/nsc/classpath/PathResolverBaseTest.scala +++ b/test/junit/scala/tools/nsc/classpath/PathResolverBaseTest.scala @@ -4,13 +4,15 @@ package scala.tools.nsc.classpath import java.io.File + import org.junit.Assert._ import org.junit._ import org.junit.rules.TemporaryFolder import org.junit.runner.RunWith import org.junit.runners.JUnit4 + import scala.tools.nsc.util.ClassPath -import scala.tools.nsc.Settings +import scala.tools.nsc.{CloseableRegistry, Settings} import scala.tools.util.PathResolver @RunWith(classOf[JUnit4]) @@ -57,7 +59,7 @@ class PathResolverBaseTest { def deleteTempDir: Unit = tempDir.delete() private def createFlatClassPath(settings: Settings) = - new PathResolver(settings).result + new PathResolver(settings, new CloseableRegistry).result @Test def testEntriesFromListOperationAgainstSeparateMethods: Unit = { diff --git a/test/junit/scala/tools/nsc/classpath/ZipAndJarFileLookupFactoryTest.scala b/test/junit/scala/tools/nsc/classpath/ZipAndJarFileLookupFactoryTest.scala index f49f04d2c56d..b58effbcfa36 100644 --- a/test/junit/scala/tools/nsc/classpath/ZipAndJarFileLookupFactoryTest.scala +++ b/test/junit/scala/tools/nsc/classpath/ZipAndJarFileLookupFactoryTest.scala @@ -14,7 +14,8 @@ class ZipAndJarFileLookupFactoryTest { Files.delete(f) val g = new scala.tools.nsc.Global(new scala.tools.nsc.Settings()) assert(!g.settings.YdisableFlatCpCaching.value) // we're testing with our JAR metadata caching enabled. - def createCp = ZipAndJarClassPathFactory.create(AbstractFile.getFile(f.toFile), g.settings) + val closeableRegistry = new CloseableRegistry + def createCp = ZipAndJarClassPathFactory.create(AbstractFile.getFile(f.toFile), g.settings, closeableRegistry) try { createZip(f, Array(), "p1/C.class") createZip(f, Array(), "p2/X.class") @@ -41,7 +42,10 @@ class ZipAndJarFileLookupFactoryTest { // And that instance should see D, not C, in package p1. assert(cp3.findClass("p1.C").isEmpty) assert(cp3.findClass("p1.D").isDefined) - } finally Files.delete(f) + } finally { + Files.delete(f) + closeableRegistry.close() + } } def createZip(zipLocation: Path, content: Array[Byte], internalPath: String): Unit = { diff --git a/test/junit/scala/tools/nsc/symtab/SymbolTableForUnitTesting.scala b/test/junit/scala/tools/nsc/symtab/SymbolTableForUnitTesting.scala index cbd5634f292f..e2b11cfecd29 100644 --- a/test/junit/scala/tools/nsc/symtab/SymbolTableForUnitTesting.scala +++ b/test/junit/scala/tools/nsc/symtab/SymbolTableForUnitTesting.scala @@ -36,7 +36,7 @@ class SymbolTableForUnitTesting extends SymbolTable { def platformPhases: List[SubComponent] = Nil - private[nsc] lazy val classPath: ClassPath = new PathResolver(settings).result + private[nsc] lazy val classPath: ClassPath = new PathResolver(settings, new CloseableRegistry).result def isMaybeBoxed(sym: Symbol): Boolean = ??? def needCompile(bin: AbstractFile, src: AbstractFile): Boolean = ???