[backport] Resource management for macro/plugin classloaders, classpath JARs

Backports:

  - scala#7366
  - scala#7644

Author: retronym

src/compiler/scala/reflect/macros/runtime/MacroRuntimes.scala

@@ -54,19 +54,8 @@ trait MacroRuntimes extends JavaReflectionRuntimes {
   /** Macro classloader that is used to resolve and run macro implementations.
    *  Loads classes from from -cp (aka the library classpath).
    *  Is also capable of detecting REPL and reusing its classloader.
-   *
-   *  When -Xmacro-jit is enabled, we sometimes fallback to on-the-fly compilation of macro implementations,
-   *  which compiles implementations into a virtual directory (very much like REPL does) and then conjures
-   *  a classloader mapped to that virtual directory.
    */
-  private lazy val defaultMacroClassloaderCache = {
-    def attemptClose(loader: ClassLoader): Unit = loader match {
-      case u: URLClassLoader => debuglog("Closing macro runtime classloader"); u.close()
-      case afcl: AbstractFileClassLoader => attemptClose(afcl.getParent)
-      case _ => ???
-    }
-    perRunCaches.newGeneric(findMacroClassLoader, attemptClose _)
-  }
+  private lazy val defaultMacroClassloaderCache: () => ClassLoader = perRunCaches.newGeneric(findMacroClassLoader())
   def defaultMacroClassloader: ClassLoader = defaultMacroClassloaderCache()
 
   /** Abstracts away resolution of macro runtimes.

src/compiler/scala/tools/nsc/CloseableRegistry.scala

@@ -0,0 +1,34 @@
+/*
+ * Scala (https://www.scala-lang.org)
+ *
+ * Copyright EPFL and Lightbend, Inc.
+ *
+ * Licensed under Apache License 2.0
+ * (http://www.apache.org/licenses/LICENSE-2.0).
+ *
+ * See the NOTICE file distributed with this work for
+ * additional information regarding copyright ownership.
+ */
+
+package scala.tools.nsc
+
+import scala.util.control.NonFatal
+
+/** Registry for resources to close when `Global` is closed */
+final class CloseableRegistry {
+  private[this] var closeables: List[java.io.Closeable] = Nil
+  final def registerClosable(c: java.io.Closeable): Unit = {
+    closeables ::= c
+  }
+
+  def close(): Unit = {
+    for (c <- closeables) {
+      try {
+        c.close()
+      } catch {
+        case NonFatal(_) =>
+      }
+    }
+    closeables = Nil
+  }
+}

src/compiler/scala/tools/nsc/GenericRunnerSettings.scala

@@ -16,7 +16,14 @@ import java.net.URL
 import scala.tools.util.PathResolver
 
 class GenericRunnerSettings(error: String => Unit) extends Settings(error) {
-  lazy val classpathURLs: Seq[URL] = new PathResolver(this).resultAsURLs
+  lazy val classpathURLs: Seq[URL] = {
+    val registry = new CloseableRegistry
+    try {
+      new PathResolver(this, registry).resultAsURLs
+    } finally {
+      registry.close()
+    }
+  }
 
   val howtorun =
     ChoiceSetting(

src/compiler/scala/tools/nsc/Global.scala

@@ -40,9 +40,11 @@ import scala.language.postfixOps
 import scala.tools.nsc.ast.{TreeGen => AstTreeGen}
 import scala.tools.nsc.classpath._
 import scala.tools.nsc.profile.Profiler
+import java.io.Closeable
 
 class Global(var currentSettings: Settings, reporter0: Reporter)
     extends SymbolTable
+    with Closeable
     with CompilationUnits
     with Plugins
     with PhaseAssembly
@@ -817,7 +819,7 @@ class Global(var currentSettings: Settings, reporter0: Reporter)
 
   /** Extend classpath of `platform` and rescan updated packages. */
   def extendCompilerClassPath(urls: URL*): Unit = {
-    val urlClasspaths = urls.map(u => ClassPathFactory.newClassPath(AbstractFile.getURL(u), settings))
+    val urlClasspaths = urls.map(u => ClassPathFactory.newClassPath(AbstractFile.getURL(u), settings, closeableRegistry))
     val newClassPath = AggregateClassPath.createAggregate(platform.classPath +: urlClasspaths : _*)
     platform.currentClassPath = Some(newClassPath)
     invalidateClassPathEntries(urls.map(_.getPath): _*)
@@ -879,7 +881,7 @@ class Global(var currentSettings: Settings, reporter0: Reporter)
       }
       entries(classPath) find matchesCanonical match {
         case Some(oldEntry) =>
-          Some(oldEntry -> ClassPathFactory.newClassPath(dir, settings))
+          Some(oldEntry -> ClassPathFactory.newClassPath(dir, settings, closeableRegistry))
         case None =>
           error(s"Error adding entry to classpath. During invalidation, no entry named $path in classpath $classPath")
           None
@@ -1706,6 +1708,13 @@ class Global(var currentSettings: Settings, reporter0: Reporter)
   }
 
   def createJavadoc    = false
+
+  final val closeableRegistry: CloseableRegistry = new CloseableRegistry
+
+  def close(): Unit = {
+    perRunCaches.clearAll()
+    closeableRegistry.close()
+  }
 }
 
 object Global {

src/compiler/scala/tools/nsc/backend/JavaPlatform.scala

@@ -27,7 +27,7 @@ trait JavaPlatform extends Platform {
   private[nsc] var currentClassPath: Option[ClassPath] = None
 
   protected[nsc] def classPath: ClassPath = {
-    if (currentClassPath.isEmpty) currentClassPath = Some(new PathResolver(settings).result)
+    if (currentClassPath.isEmpty) currentClassPath = Some(new PathResolver(settings, global.closeableRegistry).result)
     currentClassPath.get
   }
 

src/compiler/scala/tools/nsc/classpath/ClassPathFactory.scala

@@ -14,19 +14,19 @@ package scala.tools.nsc.classpath
 
 import scala.reflect.io.{AbstractFile, VirtualDirectory}
 import scala.reflect.io.Path.string2path
-import scala.tools.nsc.Settings
+import scala.tools.nsc.{CloseableRegistry, Settings}
 import FileUtils.AbstractFileOps
 import scala.tools.nsc.util.ClassPath
 
 /**
  * Provides factory methods for classpath. When creating classpath instances for a given path,
  * it uses proper type of classpath depending on a types of particular files containing sources or classes.
  */
-class ClassPathFactory(settings: Settings) {
+class ClassPathFactory(settings: Settings, closeableRegistry: CloseableRegistry) {
   /**
     * Create a new classpath based on the abstract file.
     */
-  def newClassPath(file: AbstractFile): ClassPath = ClassPathFactory.newClassPath(file, settings)
+  def newClassPath(file: AbstractFile): ClassPath = ClassPathFactory.newClassPath(file, settings, closeableRegistry)
 
   /**
     * Creators for sub classpaths which preserve this context.
@@ -70,19 +70,19 @@ class ClassPathFactory(settings: Settings) {
 
   private def createSourcePath(file: AbstractFile): ClassPath =
     if (file.isJarOrZip)
-      ZipAndJarSourcePathFactory.create(file, settings)
+      ZipAndJarSourcePathFactory.create(file, settings, closeableRegistry)
     else if (file.isDirectory)
       DirectorySourcePath(file.file)
     else
       sys.error(s"Unsupported sourcepath element: $file")
 }
 
 object ClassPathFactory {
-  def newClassPath(file: AbstractFile, settings: Settings): ClassPath = file match {
+  def newClassPath(file: AbstractFile, settings: Settings, closeableRegistry: CloseableRegistry): ClassPath = file match {
     case vd: VirtualDirectory => VirtualDirectoryClassPath(vd)
     case _ =>
       if (file.isJarOrZip)
-        ZipAndJarClassPathFactory.create(file, settings)
+        ZipAndJarClassPathFactory.create(file, settings, closeableRegistry)
       else if (file.isDirectory)
         DirectoryClassPath(file.file)
       else

src/compiler/scala/tools/nsc/classpath/DirectoryClassPath.scala

@@ -12,7 +12,7 @@
 
 package scala.tools.nsc.classpath
 
-import java.io.File
+import java.io.{Closeable, File}
 import java.net.{URI, URL}
 import java.nio.file.{FileSystems, Files, SimpleFileVisitor}
 import java.util.function.IntFunction
@@ -25,6 +25,7 @@ import FileUtils._
 import scala.collection.JavaConverters._
 import scala.collection.immutable
 import scala.reflect.internal.JDK9Reflectors
+import scala.tools.nsc.CloseableRegistry
 import scala.tools.nsc.classpath.PackageNameUtils.{packageContains, separatePkgAndClassNames}
 
 /**
@@ -61,6 +62,7 @@ trait DirectoryLookup[FileEntryType <: ClassRepresentation] extends ClassPath {
 
   private[nsc] def packages(inPackage: String): Seq[PackageEntry] = {
     val dirForPackage = getDirectory(inPackage)
+
     val nestedDirs: Array[F] = dirForPackage match {
       case None => emptyFiles
       case Some(directory) => listChildren(directory, Some(isPackage))
@@ -137,7 +139,7 @@ trait JFileDirectoryLookup[FileEntryType <: ClassRepresentation] extends Directo
 
 object JrtClassPath {
   import java.nio.file._, java.net.URI
-  def apply(release: Option[String]): Option[ClassPath] = {
+  def apply(release: Option[String], closeableRegistry: CloseableRegistry): Option[ClassPath] = {
     import scala.util.Properties._
     if (!isJavaAtLeast("9")) None
     else {
@@ -154,7 +156,11 @@ object JrtClassPath {
           try {
             val ctSym = Paths.get(javaHome).resolve("lib").resolve("ct.sym")
             if (Files.notExists(ctSym)) None
-            else Some(new CtSymClassPath(ctSym, v.toInt))
+            else {
+              val classPath = new CtSymClassPath(ctSym, v.toInt)
+              closeableRegistry.registerClosable(classPath)
+              Some(classPath)
+            }
           } catch {
             case _: Throwable => None
           }
@@ -230,7 +236,7 @@ final class JrtClassPath(fs: java.nio.file.FileSystem) extends ClassPath with No
 /**
   * Implementation `ClassPath` based on the $JAVA_HOME/lib/ct.sym backing http://openjdk.java.net/jeps/247
   */
-final class CtSymClassPath(ctSym: java.nio.file.Path, release: Int) extends ClassPath with NoSourcePaths {
+final class CtSymClassPath(ctSym: java.nio.file.Path, release: Int) extends ClassPath with NoSourcePaths with Closeable {
   import java.nio.file.Path, java.nio.file._
 
   private val fileSystem: FileSystem = FileSystems.newFileSystem(ctSym, null)
@@ -278,7 +284,7 @@ final class CtSymClassPath(ctSym: java.nio.file.Path, release: Int) extends Clas
 
   def asURLs: Seq[URL] = Nil
   def asClassPathStrings: Seq[String] = Nil
-
+  override def close(): Unit = fileSystem.close()
   def findClassFile(className: String): Option[AbstractFile] = {
     if (!className.contains(".")) None
     else {