Omnibus of -Yprofile-trace, -Ymacro-classpath, Global.close(), -Youtline, build pipelining and pickle caching

project/ScriptCommands.scala

@@ -109,11 +109,11 @@ object ScriptCommands {
     Project.setProject(session, newStructure, state)
   }
 
-  private[this] val enableOptimizer = Seq(
+  val enableOptimizer = Seq(
     scalacOptions in Compile in ThisBuild ++= Seq("-opt:l:inline", "-opt-inline-from:scala/**")
   )
 
-  private[this] val noDocs = Seq(
+  val noDocs = Seq(
     publishArtifact in (Compile, packageDoc) in ThisBuild := false
   )
 

project/headershim.scala

@@ -0,0 +1,34 @@
+package de.heikoseeberger.sbtheader
+
+import sbt._
+import sbt.Keys._
+import sbt.plugins.JvmPlugin
+
+object HeaderPlugin extends AutoPlugin {
+
+  final object autoImport {
+    class License
+    object HeaderLicense {
+      case class Custom(s: String) extends License
+    }
+    val headerLicense: SettingKey[Option[License]] = settingKey[Option[License]]("header License")
+
+    val headerSources = taskKey[scala.collection.Seq[File]]("Sources which need headers checked or created.")
+
+    val headerResources = taskKey[scala.collection.Seq[File]]("Resources which need headers checked or created.")
+
+    def headerSettings(configurations: Configuration*): Seq[Setting[_]] =
+      configurations.foldLeft(List.empty[Setting[_]]) { _ ++ inConfig(_)(toBeScopedSettings) }
+  }
+
+  import autoImport._
+
+  override def trigger = allRequirements
+
+  override def requires = JvmPlugin
+
+  override def projectSettings = headerSettings(Compile, Test)
+
+  private def toBeScopedSettings = Vector(headerSources := Nil, headerResources := Nil)
+
+}

project/plugins.sbt

@@ -29,5 +29,3 @@ libraryDependencies ++= Seq(
 concurrentRestrictions in Global := Seq(
   Tags.limitAll(1) // workaround for https://github.com/sbt/sbt/issues/2970
 )
-
-addSbtPlugin("de.heikoseeberger" % "sbt-header" % "5.0.0")

src/compiler/scala/reflect/macros/runtime/MacroRuntimes.scala

@@ -54,19 +54,8 @@ trait MacroRuntimes extends JavaReflectionRuntimes {
   /** Macro classloader that is used to resolve and run macro implementations.
    *  Loads classes from from -cp (aka the library classpath).
    *  Is also capable of detecting REPL and reusing its classloader.
-   *
-   *  When -Xmacro-jit is enabled, we sometimes fallback to on-the-fly compilation of macro implementations,
-   *  which compiles implementations into a virtual directory (very much like REPL does) and then conjures
-   *  a classloader mapped to that virtual directory.
    */
-  private lazy val defaultMacroClassloaderCache = {
-    def attemptClose(loader: ClassLoader): Unit = loader match {
-      case u: URLClassLoader => debuglog("Closing macro runtime classloader"); u.close()
-      case afcl: AbstractFileClassLoader => attemptClose(afcl.getParent)
-      case _ => ???
-    }
-    perRunCaches.newGeneric(findMacroClassLoader, attemptClose _)
-  }
+  private lazy val defaultMacroClassloaderCache: () => ClassLoader = perRunCaches.newGeneric(findMacroClassLoader())
   def defaultMacroClassloader: ClassLoader = defaultMacroClassloaderCache()
 
   /** Abstracts away resolution of macro runtimes.

src/compiler/scala/tools/nsc/CloseableRegistry.scala

@@ -0,0 +1,34 @@
+/*
+ * Scala (https://www.scala-lang.org)
+ *
+ * Copyright EPFL and Lightbend, Inc.
+ *
+ * Licensed under Apache License 2.0
+ * (http://www.apache.org/licenses/LICENSE-2.0).
+ *
+ * See the NOTICE file distributed with this work for
+ * additional information regarding copyright ownership.
+ */
+
+package scala.tools.nsc
+
+import scala.util.control.NonFatal
+
+/** Registry for resources to close when `Global` is closed */
+final class CloseableRegistry {
+  private[this] var closeables: List[java.io.Closeable] = Nil
+  final def registerClosable(c: java.io.Closeable): Unit = {
+    closeables ::= c
+  }
+
+  def close(): Unit = {
+    for (c <- closeables) {
+      try {
+        c.close()
+      } catch {
+        case NonFatal(_) =>
+      }
+    }
+    closeables = Nil
+  }
+}

src/compiler/scala/tools/nsc/CompilationUnits.scala

@@ -21,7 +21,7 @@ trait CompilationUnits { global: Global =>
   /** An object representing a missing compilation unit.
    */
   object NoCompilationUnit extends CompilationUnit(NoSourceFile) {
-    override lazy val isJava = false
+    override val isJava = false
     override def exists = false
     override def toString() = "NoCompilationUnit"
   }
@@ -153,7 +153,7 @@ trait CompilationUnits { global: Global =>
     final def comment(pos: Position, msg: String): Unit = {}
 
     /** Is this about a .java source file? */
-    lazy val isJava = source.file.name.endsWith(".java")
+    val isJava = source.file.name.endsWith(".java")
 
     override def toString() = source.toString()
   }

src/compiler/scala/tools/nsc/GenericRunnerSettings.scala

@@ -16,7 +16,14 @@ import java.net.URL
 import scala.tools.util.PathResolver
 
 class GenericRunnerSettings(error: String => Unit) extends Settings(error) {
-  lazy val classpathURLs: Seq[URL] = new PathResolver(this).resultAsURLs
+  lazy val classpathURLs: Seq[URL] = {
+    val registry = new CloseableRegistry
+    try {
+      new PathResolver(this, registry).resultAsURLs
+    } finally {
+      registry.close()
+    }
+  }
 
   val howtorun =
     ChoiceSetting(

src/compiler/scala/tools/nsc/Global.scala

@@ -40,9 +40,12 @@ import scala.language.postfixOps
 import scala.tools.nsc.ast.{TreeGen => AstTreeGen}
 import scala.tools.nsc.classpath._
 import scala.tools.nsc.profile.Profiler
+import scala.util.control.NonFatal
+import java.io.Closeable
 
 class Global(var currentSettings: Settings, reporter0: Reporter)
     extends SymbolTable
+    with Closeable
     with CompilationUnits
     with Plugins
     with PhaseAssembly
@@ -402,12 +405,16 @@ class Global(var currentSettings: Settings, reporter0: Reporter)
 
     def apply(unit: CompilationUnit): Unit
 
+    // run only the phases needed
+    protected def shouldSkipThisPhaseForJava: Boolean = {
+      this.id > (if (createJavadoc) currentRun.typerPhase.id
+      else currentRun.namerPhase.id)
+    }
+
     /** Is current phase cancelled on this unit? */
     def cancelled(unit: CompilationUnit) = {
-      // run the typer only if in `createJavadoc` mode
-      val maxJavaPhase = if (createJavadoc) currentRun.typerPhase.id else currentRun.namerPhase.id
       if (Thread.interrupted()) reporter.cancelled = true
-      reporter.cancelled || unit.isJava && this.id > maxJavaPhase
+      reporter.cancelled || unit.isJava && shouldSkipThisPhaseForJava
     }
 
     private def beforeUnit(unit: CompilationUnit): Unit = {
@@ -446,8 +453,10 @@ class Global(var currentSettings: Settings, reporter0: Reporter)
         currentRun.informUnitStarting(this, unit)
         val unit0 = currentUnit
         currentRun.currentUnit = unit
+        currentRun.profiler.beforeUnit(phase, unit.source.file)
         try apply(unit)
         finally {
+          currentRun.profiler.afterUnit(phase, unit.source.file)
           currentRun.currentUnit = unit0
           currentRun.advanceUnit()
         }
@@ -817,7 +826,7 @@ class Global(var currentSettings: Settings, reporter0: Reporter)
 
   /** Extend classpath of `platform` and rescan updated packages. */
   def extendCompilerClassPath(urls: URL*): Unit = {
-    val urlClasspaths = urls.map(u => ClassPathFactory.newClassPath(AbstractFile.getURL(u), settings))
+    val urlClasspaths = urls.map(u => ClassPathFactory.newClassPath(AbstractFile.getURL(u), settings, closeableRegistry))
     val newClassPath = AggregateClassPath.createAggregate(platform.classPath +: urlClasspaths : _*)
     platform.currentClassPath = Some(newClassPath)
     invalidateClassPathEntries(urls.map(_.getPath): _*)
@@ -879,7 +888,7 @@ class Global(var currentSettings: Settings, reporter0: Reporter)
       }
       entries(classPath) find matchesCanonical match {
         case Some(oldEntry) =>
-          Some(oldEntry -> ClassPathFactory.newClassPath(dir, settings))
+          Some(oldEntry -> ClassPathFactory.newClassPath(dir, settings, closeableRegistry))
         case None =>
           error(s"Error adding entry to classpath. During invalidation, no entry named $path in classpath $classPath")
           None
@@ -1110,6 +1119,9 @@ class Global(var currentSettings: Settings, reporter0: Reporter)
 
   def newJavaUnitParser(unit: CompilationUnit): JavaUnitParser = new JavaUnitParser(unit)
 
+  override protected[scala] def currentRunProfilerBeforeCompletion(root: Symbol, associatedFile: AbstractFile): Unit = currentRun.profiler.beforeCompletion(root, associatedFile)
+  override protected[scala] def currentRunProfilerAfterCompletion(root: Symbol, associatedFile: AbstractFile): Unit = currentRun.profiler.afterCompletion(root, associatedFile)
+
   /** A Run is a single execution of the compiler on a set of units.
    */
   class Run extends RunContextApi with RunReporting with RunParsing {
@@ -1438,7 +1450,20 @@ class Global(var currentSettings: Settings, reporter0: Reporter)
 
     private def printArgs(sources: List[SourceFile]): Unit = {
       if (settings.printArgs.isSetByUser) {
-        val argsFile = (settings.recreateArgs ::: sources.map(_.file.absolute.toString())).mkString("", "\n", "\n")
+        val singleOuputDir: List[String] = if (settings.d.value == settings.d.default) {
+          settings.outputDirs.getSingleOutput match {
+            case Some(file) =>
+              val jfile = file.file
+              if (jfile != null && !java.nio.file.Files.isSameFile(jfile.toPath, java.nio.file.Paths.get(settings.d.value))) {
+                // A build tool must have used `settings.outDirs.setSingleOutput`, bypassing `-d`.
+                // Render that to the equivalent -d arguments.
+                "-d" :: jfile.toString :: Nil
+              } else Nil
+            case _ => Nil
+          }
+        } else Nil
+        val recreated = settings.userSetSettings.toList.filterNot(_ eq settings.printArgs).flatMap(_.unparse)
+        val argsFile = (recreated ::: singleOuputDir ::: sources.map(_.file.absolute.toString())).mkString("", "\n", "\n")
         settings.printArgs.value match {
           case "-" =>
             reporter.echo(argsFile)
@@ -1474,7 +1499,7 @@ class Global(var currentSettings: Settings, reporter0: Reporter)
     private final val GlobalPhaseName = "global (synthetic)"
     protected final val totalCompileTime = statistics.newTimer("#total compile time", GlobalPhaseName)
 
-    def compileUnits(units: List[CompilationUnit], fromPhase: Phase): Unit =  compileUnitsInternal(units,fromPhase)
+    def compileUnits(units: List[CompilationUnit], fromPhase: Phase): Unit = compileUnitsInternal(units,fromPhase)
     private def compileUnitsInternal(units: List[CompilationUnit], fromPhase: Phase) {
       units foreach addUnit
       reporter.reset()
@@ -1701,6 +1726,13 @@ class Global(var currentSettings: Settings, reporter0: Reporter)
   }
 
   def createJavadoc    = false
+
+  final val closeableRegistry: CloseableRegistry = new CloseableRegistry
+
+  def close(): Unit = {
+    perRunCaches.clearAll()
+    closeableRegistry.close()
+  }
 }
 
 object Global {

src/compiler/scala/tools/nsc/PickleExtractor.scala

@@ -0,0 +1,121 @@
+package scala.tools.nsc
+
+import java.io.Closeable
+import java.nio.file.attribute.BasicFileAttributes
+import java.nio.file.{FileVisitResult, Files, Path, SimpleFileVisitor, _}
+
+import scala.collection.JavaConverters.{asScalaBufferConverter, bufferAsJavaListConverter, collectionAsScalaIterableConverter}
+import scala.reflect.internal.pickling.ByteCodecs
+import scala.reflect.io.RootPath
+import scala.tools.asm.tree.ClassNode
+import scala.tools.asm.{ClassReader, ClassWriter, Opcodes}
+
+object PickleExtractor {
+
+  def main(args: Array[String]): Unit = {
+    args.toList match {
+      case input :: output :: Nil =>
+        process(Paths.get(input), Paths.get(output))
+      case _ =>
+    }
+  }
+  def process(input: Path, output: Path): Unit = {
+    val inputPath = RootPath(input, writable = false)
+    val outputPath = RootPath(output, writable = true)
+    try {
+      val root = inputPath.root
+      Files.createDirectories(outputPath.root)
+      val visitor = new SimpleFileVisitor[Path] {
+        override def preVisitDirectory(dir: Path, attrs: BasicFileAttributes): FileVisitResult = {
+          if (dir != root) {
+            val outputDir = outputPath.root.resolve(root.relativize(dir).toString)
+            Files.createDirectories(outputDir)
+          }
+          FileVisitResult.CONTINUE
+        }
+        override def visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult = {
+          if (file.getFileName.toString.endsWith(".class")) {
+            stripClassFile(Files.readAllBytes(file)) match {
+              case Class(out) =>
+                Files.write(outputPath.root.resolve(root.relativize(file).toString), out)
+              case Pickle(out) =>
+                Files.write(outputPath.root.resolve(root.relativize(file).toString.replaceAll(".class$", ".sig")), out)
+              case Skip =>
+            }
+          }
+          FileVisitResult.CONTINUE
+        }
+      }
+      Files.walkFileTree(root, visitor)
+    } finally {
+      inputPath.close()
+      outputPath.close()
+    }
+  }
+
+  def stripClassFile(classfile: Array[Byte]): OutputFile = {
+    val input = new ClassNode()
+    new ClassReader(classfile).accept(input, ClassReader.SKIP_DEBUG | ClassReader.SKIP_FRAMES | ClassReader.SKIP_CODE)
+    var output = new ClassNode()
+    output.name = input.name
+    output.access = input.access
+    output.version = input.version
+
+    var foundScalaSig = false
+
+    def isScalaAnnotation(desc: String) = (desc == "Lscala/reflect/ScalaSignature;" || desc == "Lscala/reflect/ScalaLongSignature;") && {
+      foundScalaSig = true
+
+      true
+    }
+
+    var pickleData: Array[Byte] = null
+    if (input.visibleAnnotations != null) {
+      input.visibleAnnotations.asScala.foreach { node =>
+        if (node.desc == "Lscala/reflect/ScalaSignature;") {
+          val Array("bytes", data: String) = node.values.toArray()
+          val bytes = data.getBytes(java.nio.charset.StandardCharsets.UTF_8)
+          val len = ByteCodecs.decode(bytes)
+          pickleData = bytes.take(len)
+        } else if (node.desc == "Lscala/reflect/ScalaLongSignature;") {
+          val Array("bytes", data: java.util.Collection[String @unchecked]) = node.values.toArray()
+          val encoded = data.asScala.toArray flatMap (_.getBytes(java.nio.charset.StandardCharsets.UTF_8))
+          val len = ByteCodecs.decode(encoded)
+          pickleData = encoded.take(len)
+        }
+      }
+      output.visibleAnnotations = input.visibleAnnotations.asScala.filter(node => isScalaAnnotation(node.desc) && {
+        true
+      }).asJava
+    }
+    var foundScalaAttr = false
+    if (input.attrs != null) {
+      output.attrs = input.attrs.asScala.filter(attr => (attr.`type` == "Scala" || attr.`type` == "ScalaSig") && {
+        foundScalaAttr = true;
+        true
+      }).asJava
+    }
+    val writer = new ClassWriter(Opcodes.ASM7_EXPERIMENTAL)
+    val isScalaRaw = foundScalaAttr && !foundScalaSig
+    if (isScalaRaw) Skip
+    else {
+      if (pickleData == null) {
+        output = input
+        output.accept(writer)
+        Class(writer.toByteArray)
+      } else {
+        output.accept(writer)
+        Pickle(pickleData)
+      }
+    }
+  }
+
+  sealed abstract class OutputFile
+
+  case object Skip extends OutputFile
+
+  case class Class(content: Array[Byte]) extends OutputFile
+
+  case class Pickle(content: Array[Byte]) extends OutputFile
+
+}