diff --git a/src/compiler/scala/tools/nsc/PipelineMain.scala b/src/compiler/scala/tools/nsc/PipelineMain.scala index 29b9c560bcec..44f46cbc9366 100644 --- a/src/compiler/scala/tools/nsc/PipelineMain.scala +++ b/src/compiler/scala/tools/nsc/PipelineMain.scala @@ -17,6 +17,7 @@ import java.lang.Thread.UncaughtExceptionHandler import java.nio.file.attribute.FileTime import java.nio.file.{Files, Path, Paths} import java.time.Instant +import java.util.concurrent.ConcurrentHashMap import java.util.{Collections, Locale} import java.util.concurrent.atomic.{AtomicBoolean, AtomicInteger} @@ -44,10 +45,13 @@ class PipelineMainClass(argFiles: Seq[Path], pipelineSettings: PipelineMain.Pipe val root = file.getRoot // An empty component on Unix, just the drive letter on Windows val validRootPathComponent = root.toString.replaceAllLiterally("/", "").replaceAllLiterally(":", "") - changeExtension(pickleCache.resolve(validRootPathComponent).resolve(root.relativize(file)).normalize(), newExtension) + val result = changeExtension(pickleCache.resolve(validRootPathComponent).resolve(root.relativize(file)).normalize(), newExtension) + if (useJars) Files.createDirectories(result.getParent) + strippedAndExportedClassPath.put(file.toRealPath().normalize(), result) + result } - private val strippedAndExportedClassPath = mutable.HashMap[Path, Path]() + private val strippedAndExportedClassPath = new ConcurrentHashMap[Path, Path]().asScala /** Forward errors to the (current) reporter. */ protected def scalacError(msg: String): Unit = { @@ -73,51 +77,6 @@ class PipelineMainClass(argFiles: Seq[Path], pipelineSettings: PipelineMain.Pipe p.getParent.resolve(changedFileName) } - def registerPickleClassPath[G <: Global](output: Path, data: mutable.AnyRefMap[G#Symbol, PickleBuffer]): Unit = { - val jarPath = cachePath(output) - val root = RootPath(jarPath, writable = true) - Files.createDirectories(root.root) - - val dirs = mutable.Map[G#Symbol, Path]() - def packageDir(packSymbol: G#Symbol): Path = { - if (packSymbol.isEmptyPackageClass) root.root - else if (dirs.contains(packSymbol)) dirs(packSymbol) - else if (packSymbol.owner.isRoot) { - val subDir = root.root.resolve(packSymbol.encodedName) - Files.createDirectories(subDir) - dirs.put(packSymbol, subDir) - subDir - } else { - val base = packageDir(packSymbol.owner) - val subDir = base.resolve(packSymbol.encodedName) - Files.createDirectories(subDir) - dirs.put(packSymbol, subDir) - subDir - } - } - val written = new java.util.IdentityHashMap[AnyRef, Unit]() - try { - for ((symbol, pickle) <- data) { - if (!written.containsKey(pickle)) { - val base = packageDir(symbol.owner) - val primary = base.resolve(symbol.encodedName + ".sig") - val writer = new BufferedOutputStream(Files.newOutputStream(primary)) - try { - writer.write(pickle.bytes, 0, pickle.writeIndex) - } finally { - writer.close() - } - written.put(pickle, ()) - } - } - } finally { - root.close() - } - Files.setLastModifiedTime(jarPath, FileTime.from(Instant.now())) - strippedAndExportedClassPath.put(output.toRealPath().normalize(), jarPath) - } - - def writeDotFile(logDir: Path, dependsOn: mutable.LinkedHashMap[Task, List[Dependency]]): Unit = { val builder = new java.lang.StringBuilder() builder.append("digraph projects {\n") @@ -375,7 +334,6 @@ class PipelineMainClass(argFiles: Seq[Path], pipelineSettings: PipelineMain.Pipe if (p.outlineTimer.durationMicros > 0d) { val desc = if (strategy == OutlineTypePipeline) "outline-type" else "parser-to-pickler" events += durationEvent(p.label, desc, p.outlineTimer) - events += durationEvent(p.label, "pickle-export", p.pickleExportTimer) } for ((g, ix) <- p.groups.zipWithIndex) { if (g.timer.durationMicros > 0d) @@ -453,7 +411,6 @@ class PipelineMainClass(argFiles: Seq[Path], pipelineSettings: PipelineMain.Pipe val isGrouped = groups.size > 1 val outlineTimer = new Timer() - val pickleExportTimer = new Timer val javaTimer = new Timer() var outlineCriticalPathMs = 0d @@ -491,14 +448,11 @@ class PipelineMainClass(argFiles: Seq[Path], pipelineSettings: PipelineMain.Pipe command.settings.Youtline.value = true command.settings.stopAfter.value = List("pickler") command.settings.Ymacroexpand.value = command.settings.MacroExpand.None + command.settings.YpickleWrite.value = cachePath(command.settings.outputDirs.getSingleOutput.get.file.toPath).toAbsolutePath.toString val run1 = new compiler.Run() run1 compile files outlineTimer.stop() log(f"scalac outline: done ${outlineTimer.durationMs}%.0f ms") - pickleExportTimer.start() - registerPickleClassPath(command.settings.outputDirs.getSingleOutput.get.file.toPath, run1.symData) - pickleExportTimer.stop() - log(f"scalac: exported pickles ${pickleExportTimer.durationMs}%.0f ms") reporter.finish() if (reporter.hasErrors) { log("scalac outline: failed") @@ -518,6 +472,7 @@ class PipelineMainClass(argFiles: Seq[Path], pipelineSettings: PipelineMain.Pipe command.settings.Youtline.value = false command.settings.stopAfter.value = Nil command.settings.Ymacroexpand.value = command.settings.MacroExpand.Normal + command.settings.YpickleWrite.value = "" val groupCount = groups.size for ((group, ix) <- groups.zipWithIndex) { @@ -552,18 +507,14 @@ class PipelineMainClass(argFiles: Seq[Path], pipelineSettings: PipelineMain.Pipe assert(groups.size == 1) val group = groups.head log("scalac: start") + command.settings.YpickleWrite.value = cachePath(command.settings.outputDirs.getSingleOutput.get.file.toPath).toString outlineTimer.start() try { val run2 = new compiler.Run() { - override def advancePhase(): Unit = { if (compiler.phase == this.picklerPhase) { outlineTimer.stop() log(f"scalac outline: done ${outlineTimer.durationMs}%.0f ms") - pickleExportTimer.start() - registerPickleClassPath(command.settings.outputDirs.getSingleOutput.get.file.toPath, symData) - pickleExportTimer.stop() - log(f"scalac: exported pickles ${pickleExportTimer.durationMs}%.0f ms") outlineDone.complete(Success(())) group.timer.start() } diff --git a/src/compiler/scala/tools/nsc/backend/jvm/ClassfileWriters.scala b/src/compiler/scala/tools/nsc/backend/jvm/ClassfileWriters.scala index 8109add34c40..629316fed6b0 100644 --- a/src/compiler/scala/tools/nsc/backend/jvm/ClassfileWriters.scala +++ b/src/compiler/scala/tools/nsc/backend/jvm/ClassfileWriters.scala @@ -23,6 +23,7 @@ import java.util.concurrent.ConcurrentHashMap import java.util.zip.{CRC32, Deflater, ZipEntry, ZipOutputStream} import scala.reflect.internal.util.{NoPosition, Statistics} +import scala.reflect.io.{PlainNioFile, VirtualFile} import scala.tools.nsc.Global import scala.tools.nsc.backend.jvm.BTypes.InternalName import scala.tools.nsc.io.AbstractFile @@ -44,12 +45,15 @@ abstract class ClassfileWriters { /** * Write a classfile */ - def write(name: InternalName, bytes: Array[Byte], paths: CompilationUnitPaths) + def writeClass(name: InternalName, bytes: Array[Byte], sourceFile: AbstractFile) /** * Close the writer. Behavior is undefined after a call to `close`. */ def close(): Unit + + protected def classRelativePath(className: InternalName, suffix: String = ".class"): Path = + Paths.get(className.replace('.', '/') + suffix) } object ClassfileWriter { @@ -68,125 +72,173 @@ abstract class ClassfileWriters { } } - def singleWriter(file: AbstractFile): UnderlyingClassfileWriter = { - if (file hasExtension "jar") { - new JarClassWriter(file, jarManifestMainClass, settings.YjarCompressionLevel.value) - } else if (file.isVirtual) { - new VirtualClassWriter() - } else if (file.isDirectory) { - new DirClassWriter() - } else { - throw new IllegalStateException(s"don't know how to handle an output of $file [${file.getClass}]") - } - } - val basicClassWriter = settings.outputDirs.getSingleOutput match { - case Some(dest) => singleWriter(dest) + case Some(dest) => new SingleClassWriter(FileWriter(global, dest, jarManifestMainClass)) case None => val distinctOutputs: Set[AbstractFile] = settings.outputDirs.outputs.map(_._2)(scala.collection.breakOut) - if (distinctOutputs.size == 1) singleWriter(distinctOutputs.head) - else new MultiClassWriter(distinctOutputs.map { output: AbstractFile => output -> singleWriter(output) }(scala.collection.breakOut)) + if (distinctOutputs.size == 1) new SingleClassWriter(FileWriter(global, distinctOutputs.head, jarManifestMainClass)) + else { + val sourceToOutput: Map[AbstractFile, AbstractFile] = global.currentRun.units.map(unit => (unit.source.file, frontendAccess.compilerSettings.outputDirectory(unit.source.file))).toMap + new MultiClassWriter(sourceToOutput, distinctOutputs.map { output: AbstractFile => output -> FileWriter(global, output, jarManifestMainClass) }(scala.collection.breakOut)) + } } val withAdditionalFormats = if (settings.Ygenasmp.valueSetByUser.isEmpty && settings.Ydumpclasses.valueSetByUser.isEmpty) basicClassWriter else { - val asmp = settings.Ygenasmp.valueSetByUser map { dir: String => new AsmClassWriter(getDirectory(dir)) } - val dump = settings.Ydumpclasses.valueSetByUser map { dir: String => new DumpClassWriter(getDirectory(dir)) } - new AllClassWriter(basicClassWriter, asmp, dump) + val asmp = settings.Ygenasmp.valueSetByUser map { dir: String => FileWriter(global, new PlainNioFile(getDirectory(dir)), None) } + val dump = settings.Ydumpclasses.valueSetByUser map { dir: String => FileWriter(global, new PlainNioFile(getDirectory(dir)), None) } + new DebugClassWriter(basicClassWriter, asmp, dump) } val enableStats = statistics.enabled && settings.YaddBackendThreads.value == 1 if (enableStats) new WithStatsWriter(withAdditionalFormats) else withAdditionalFormats } - /** - * A marker trait for Classfilewriters that actually write, rather than layer functionality - */ - sealed trait UnderlyingClassfileWriter extends ClassfileWriter - - private final class JarClassWriter(file: AbstractFile, mainClass: Option[String], compressionLevel: Int) extends UnderlyingClassfileWriter { - //keep these imports local - avoid confusion with scala naming - import java.util.jar.Attributes.Name - import java.util.jar.{JarOutputStream, Manifest} - - val storeOnly = compressionLevel == Deflater.NO_COMPRESSION - - val jarWriter: JarOutputStream = { - val manifest = new Manifest() - mainClass foreach { c => manifest.getMainAttributes.put(Name.MAIN_CLASS, c) } - val jar = new JarOutputStream(new BufferedOutputStream(new FileOutputStream(file.file), 64000), manifest) - jar.setLevel(compressionLevel) - if (storeOnly) jar.setMethod(ZipOutputStream.STORED) - jar + /** Writes to the output directory corresponding to the source file, if multiple output directories are specified */ + private final class MultiClassWriter(sourceToOutput: Map[AbstractFile, AbstractFile], underlying: Map[AbstractFile, FileWriter]) extends ClassfileWriter { + private def getUnderlying(sourceFile: AbstractFile, outputDir: AbstractFile) = underlying.getOrElse(outputDir, { + throw new Exception(s"Cannot determine output directory for ${sourceFile} with output ${outputDir}. Configured outputs are ${underlying.keySet}") + }) + + override def writeClass(className: InternalName, bytes: Array[Byte], sourceFile: AbstractFile): Unit = { + getUnderlying(sourceFile, sourceToOutput(sourceFile)).writeFile(classRelativePath(className), bytes) } + override def close(): Unit = underlying.values.foreach(_.close()) + } + private final class SingleClassWriter(underlying: FileWriter) extends ClassfileWriter { + override def writeClass(className: InternalName, bytes: Array[Byte], sourceFile: AbstractFile): Unit = { + underlying.writeFile(classRelativePath(className), bytes) + } + override def close(): Unit = underlying.close() + } - lazy val crc = new CRC32 - - override def write(className: InternalName, bytes: Array[Byte], paths: CompilationUnitPaths): Unit = this.synchronized { - val path = className + ".class" - val entry = new ZipEntry(path) - if (storeOnly) { - // When using compression method `STORED`, the ZIP spec requires the CRC and compressed/ - // uncompressed sizes to be written before the data. The JarOutputStream could compute the - // values while writing the data, but not patch them into the stream after the fact. So we - // need to pre-compute them here. The compressed size is taken from size. - // https://stackoverflow.com/questions/1206970/how-to-create-uncompressed-zip-archive-in-java/5868403 - // With compression method `DEFLATED` JarOutputStream computes and sets the values. - entry.setSize(bytes.length) - crc.reset() - crc.update(bytes) - entry.setCrc(crc.getValue) + private final class DebugClassWriter(basic: ClassfileWriter, asmp: Option[FileWriter], dump: Option[FileWriter]) extends ClassfileWriter { + override def writeClass(className: InternalName, bytes: Array[Byte], sourceFile: AbstractFile): Unit = { + basic.writeClass(className, bytes, sourceFile) + asmp.foreach { writer => + val asmBytes = AsmUtils.textify(AsmUtils.readClass(bytes)).getBytes(StandardCharsets.UTF_8) + writer.writeFile(classRelativePath(className, ".asm"), asmBytes) } - jarWriter.putNextEntry(entry) - try jarWriter.write(bytes, 0, bytes.length) - finally jarWriter.flush() + dump.foreach { writer => + writer.writeFile(classRelativePath(className), bytes) + } + } + + override def close(): Unit = { + basic.close() + asmp.foreach(_.close()) + dump.foreach(_.close()) + } + } + + private final class WithStatsWriter(underlying: ClassfileWriter) extends ClassfileWriter { + override def writeClass(className: InternalName, bytes: Array[Byte], sourceFile: AbstractFile): Unit = { + val statistics = frontendAccess.unsafeStatistics + val snap = statistics.startTimer(statistics.bcodeWriteTimer) + try underlying.writeClass(className, bytes, sourceFile) + finally statistics.stopTimer(statistics.bcodeWriteTimer, snap) } - override def close(): Unit = this.synchronized(jarWriter.close()) + override def close(): Unit = underlying.close() } + } - private sealed class DirClassWriter extends UnderlyingClassfileWriter { - val builtPaths = new ConcurrentHashMap[Path, java.lang.Boolean]() - val noAttributes = Array.empty[FileAttribute[_]] - private val isWindows = scala.util.Properties.isWin + sealed trait FileWriter { + def writeFile(relativePath: Path, bytes: Array[Byte]): Unit + def close(): Unit + } - def ensureDirForPath(baseDir: Path, filePath: Path): Unit = { - import java.lang.Boolean.TRUE - val parent = filePath.getParent - if (!builtPaths.containsKey(parent)) { - try Files.createDirectories(parent, noAttributes: _*) - catch { - case e: FileAlreadyExistsException => - // `createDirectories` reports this exception if `parent` is an existing symlink to a directory - // but that's fine for us (and common enough, `scalac -d /tmp` on mac targets symlink). - if (!Files.isDirectory(parent)) - throw new FileConflictException(s"Can't create directory $parent; there is an existing (non-directory) file in its path", e) - } - builtPaths.put(baseDir, TRUE) - var current = parent - while ((current ne null) && (null ne builtPaths.put(current, TRUE))) { - current = current.getParent - } - } + object FileWriter { + def apply(global: Global, file: AbstractFile, jarManifestMainClass: Option[String]): FileWriter = { + if (file hasExtension "jar") { + val jarCompressionLevel = global.settings.YjarCompressionLevel.value + new JarEntryWriter(file, jarManifestMainClass, jarCompressionLevel) + } else if (file.isVirtual) { + new VirtualFileWriter(file) + } else if (file.isDirectory) { + new DirEntryWriter(file.file.toPath) + } else { + throw new IllegalStateException(s"don't know how to handle an output of $file [${file.getClass}]") } + } + } - protected def getPath(className: InternalName, paths: CompilationUnitPaths) = paths.outputPath.resolve(className + ".class") + private final class JarEntryWriter(file: AbstractFile, mainClass: Option[String], compressionLevel: Int) extends FileWriter { + //keep these imports local - avoid confusion with scala naming + import java.util.jar.Attributes.Name + import java.util.jar.{JarOutputStream, Manifest} - protected def formatData(rawBytes: Array[Byte]) = rawBytes + val storeOnly = compressionLevel == Deflater.NO_COMPRESSION - protected def qualifier: String = "" + val jarWriter: JarOutputStream = { + val manifest = new Manifest() + mainClass foreach { c => manifest.getMainAttributes.put(Name.MAIN_CLASS, c) } + val jar = new JarOutputStream(new BufferedOutputStream(new FileOutputStream(file.file), 64000), manifest) + jar.setLevel(compressionLevel) + if (storeOnly) jar.setMethod(ZipOutputStream.STORED) + jar + } - // the common case is that we are are creating a new file, and on MS Windows the create and truncate is expensive - // because there is not an options in the windows API that corresponds to this so the truncate is applied as a separate call - // even if the file is new. - // as this is rare, its best to always try to create a new file, and it that fails, then open with truncate if that fails + lazy val crc = new CRC32 + + override def writeFile(relativePath: Path, bytes: Array[Byte]): Unit = this.synchronized { + val entry = new ZipEntry(relativePath.toString) + if (storeOnly) { + // When using compression method `STORED`, the ZIP spec requires the CRC and compressed/ + // uncompressed sizes to be written before the data. The JarOutputStream could compute the + // values while writing the data, but not patch them into the stream after the fact. So we + // need to pre-compute them here. The compressed size is taken from size. + // https://stackoverflow.com/questions/1206970/how-to-create-uncompressed-zip-archive-in-java/5868403 + // With compression method `DEFLATED` JarOutputStream computes and sets the values. + entry.setSize(bytes.length) + crc.reset() + crc.update(bytes) + entry.setCrc(crc.getValue) + } + jarWriter.putNextEntry(entry) + try jarWriter.write(bytes, 0, bytes.length) + finally jarWriter.flush() + } - private val fastOpenOptions = util.EnumSet.of(StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE) - private val fallbackOpenOptions = util.EnumSet.of(StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING) + override def close(): Unit = this.synchronized(jarWriter.close()) + } - override def write(className: InternalName, rawBytes: Array[Byte], paths: CompilationUnitPaths): Unit = try { - val path = getPath(className, paths) - val bytes = formatData(rawBytes) - ensureDirForPath(paths.outputPath, path) + private final class DirEntryWriter(base: Path) extends FileWriter { + val builtPaths = new ConcurrentHashMap[Path, java.lang.Boolean]() + val noAttributes = Array.empty[FileAttribute[_]] + private val isWindows = scala.util.Properties.isWin + + def ensureDirForPath(baseDir: Path, filePath: Path): Unit = { + import java.lang.Boolean.TRUE + val parent = filePath.getParent + if (!builtPaths.containsKey(parent)) { + try Files.createDirectories(parent, noAttributes: _*) + catch { + case e: FileAlreadyExistsException => + // `createDirectories` reports this exception if `parent` is an existing symlink to a directory + // but that's fine for us (and common enough, `scalac -d /tmp` on mac targets symlink). + if (!Files.isDirectory(parent)) + throw new FileConflictException(s"Can't create directory $parent; there is an existing (non-directory) file in its path", e) + } + builtPaths.put(baseDir, TRUE) + var current = parent + while ((current ne null) && (null ne builtPaths.put(current, TRUE))) { + current = current.getParent + } + } + } + + // the common case is that we are are creating a new file, and on MS Windows the create and truncate is expensive + // because there is not an options in the windows API that corresponds to this so the truncate is applied as a separate call + // even if the file is new. + // as this is rare, its best to always try to create a new file, and it that fails, then open with truncate if that fails + + private val fastOpenOptions = util.EnumSet.of(StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE) + private val fallbackOpenOptions = util.EnumSet.of(StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING) + + override def writeFile(relativePath: Path, bytes: Array[Byte]): Unit = { + val path = base.resolve(relativePath) + try { + ensureDirForPath(base, path) val os = if (isWindows) { try FileChannel.open(path, fastOpenOptions) catch { @@ -208,95 +260,38 @@ abstract class ClassfileWriters { os.close() } catch { case e: FileConflictException => - frontendAccess.backendReporting.error(NoPosition, s"error writing $className$qualifier: ${e.getMessage}") + frontendAccess.backendReporting.error(NoPosition, s"error writing $path: ${e.getMessage}") case e: java.nio.file.FileSystemException => if (frontendAccess.compilerSettings.debug) e.printStackTrace() - frontendAccess.backendReporting.error(NoPosition, s"error writing $className$qualifier: ${e.getClass.getName} ${e.getMessage}") - + frontendAccess.backendReporting.error(NoPosition, s"error writing $path: ${e.getClass.getName} ${e.getMessage}") } - - override def close(): Unit = () - } - - private final class AsmClassWriter(asmOutputPath: Path) extends DirClassWriter { - override protected def getPath(className: InternalName, paths: CompilationUnitPaths) = asmOutputPath.resolve(className + ".asmp") - - override protected def formatData(rawBytes: Array[Byte]) = AsmUtils.textify(AsmUtils.readClass(rawBytes)).getBytes(StandardCharsets.UTF_8) - - override protected def qualifier: String = " [for asmp]" } - private final class DumpClassWriter(dumpOutputPath: Path) extends DirClassWriter { - override protected def getPath(className: InternalName, paths: CompilationUnitPaths) = dumpOutputPath.resolve(className + ".class") - - override protected def qualifier: String = " [for dump]" - } - - private final class VirtualClassWriter extends UnderlyingClassfileWriter { - private def getFile(base: AbstractFile, clsName: String, suffix: String): AbstractFile = { - def ensureDirectory(dir: AbstractFile): AbstractFile = - if (dir.isDirectory) dir - else throw new FileConflictException(s"${base.path}/$clsName$suffix: ${dir.path} is not a directory") - - var dir = base - val pathParts = clsName.split("[./]").toList - for (part <- pathParts.init) dir = ensureDirectory(dir) subdirectoryNamed part - ensureDirectory(dir) fileNamed pathParts.last + suffix - } - - private def writeBytes(outFile: AbstractFile, bytes: Array[Byte]): Unit = { - val out = new DataOutputStream(outFile.bufferedOutput) - try out.write(bytes, 0, bytes.length) - finally out.close() - } - - override def write(className: InternalName, bytes: Array[Byte], paths: CompilationUnitPaths): Unit = { - val outFile = getFile(paths.outputDir, className, ".class") - writeBytes(outFile, bytes) - } - - override def close(): Unit = () - } - - private final class MultiClassWriter(underlying: Map[AbstractFile, UnderlyingClassfileWriter]) extends ClassfileWriter { - private def getUnderlying(paths: CompilationUnitPaths) = underlying.getOrElse(paths.outputDir, { - throw new Exception(s"Cannot determine output directory for ${paths.sourceFile} with output ${paths.outputDir}. Configured outputs are ${underlying.keySet}") - }) - - override def write(className: InternalName, bytes: Array[Byte], paths: CompilationUnitPaths): Unit = { - getUnderlying(paths).write(className, bytes, paths) - } + override def close(): Unit = () + } - override def close(): Unit = underlying.values.foreach(_.close()) + private final class VirtualFileWriter(base: AbstractFile) extends FileWriter { + private def getFile(base: AbstractFile, path: Path): AbstractFile = { + def ensureDirectory(dir: AbstractFile): AbstractFile = + if (dir.isDirectory) dir + else throw new FileConflictException(s"${base.path}/${path}: ${dir.path} is not a directory") + var dir = base + for (i <- 0 until path.getNameCount - 1) dir = ensureDirectory(dir) subdirectoryNamed path.getName(i).toString + ensureDirectory(dir) fileNamed path.getFileName.toString } - private final class AllClassWriter(basic: ClassfileWriter, asmp: Option[UnderlyingClassfileWriter], dump: Option[UnderlyingClassfileWriter]) extends ClassfileWriter { - override def write(className: InternalName, bytes: Array[Byte], paths: CompilationUnitPaths): Unit = { - basic.write(className, bytes, paths) - asmp.foreach(_.write(className, bytes, paths)) - dump.foreach(_.write(className, bytes, paths)) - } - - override def close(): Unit = { - basic.close() - asmp.foreach(_.close()) - dump.foreach(_.close()) - } + private def writeBytes(outFile: AbstractFile, bytes: Array[Byte]): Unit = { + val out = new DataOutputStream(outFile.bufferedOutput) + try out.write(bytes, 0, bytes.length) + finally out.close() } - private final class WithStatsWriter(underlying: ClassfileWriter) - extends ClassfileWriter { - override def write(className: InternalName, bytes: Array[Byte], paths: CompilationUnitPaths): Unit = { - val statistics = frontendAccess.unsafeStatistics - val snap = statistics.startTimer(statistics.bcodeWriteTimer) - underlying.write(className, bytes, paths) - statistics.stopTimer(statistics.bcodeWriteTimer, snap) - } - - override def close(): Unit = underlying.close() + override def writeFile(relativePath: Path, bytes: Array[Byte]): Unit = { + val outFile = getFile(base, relativePath) + writeBytes(outFile, bytes) } - + override def close(): Unit = () } /** Can't output a file due to the state of the file system. */ diff --git a/src/compiler/scala/tools/nsc/backend/jvm/GeneratedClassHandler.scala b/src/compiler/scala/tools/nsc/backend/jvm/GeneratedClassHandler.scala index ae7d772bd629..ce02b31a1a58 100644 --- a/src/compiler/scala/tools/nsc/backend/jvm/GeneratedClassHandler.scala +++ b/src/compiler/scala/tools/nsc/backend/jvm/GeneratedClassHandler.scala @@ -108,8 +108,7 @@ private[jvm] object GeneratedClassHandler { private val processingUnits = ListBuffer.empty[CompilationUnitInPostProcess] def process(unit: GeneratedCompilationUnit): Unit = { - val unitInPostProcess = new CompilationUnitInPostProcess(unit.classes, - CompilationUnitPaths(unit.sourceFile, frontendAccess.compilerSettings.outputDirectory(unit.sourceFile))) + val unitInPostProcess = new CompilationUnitInPostProcess(unit.classes, unit.sourceFile) postProcessUnit(unitInPostProcess) processingUnits += unitInPostProcess } @@ -122,7 +121,7 @@ private[jvm] object GeneratedClassHandler { // we 'take' classes to reduce the memory pressure // as soon as the class is consumed and written, we release its data unitInPostProcess.takeClasses() foreach { - postProcessor.sendToDisk(_, unitInPostProcess.paths) + postProcessor.sendToDisk(_, unitInPostProcess.sourceFile) } } } @@ -169,7 +168,7 @@ private[jvm] object GeneratedClassHandler { case _: ClosedByInterruptException => throw new InterruptedException() case NonFatal(t) => t.printStackTrace() - frontendAccess.backendReporting.error(NoPosition, s"unable to write ${unitInPostProcess.paths.sourceFile} $t") + frontendAccess.backendReporting.error(NoPosition, s"unable to write ${unitInPostProcess.sourceFile} $t") } } } @@ -198,18 +197,13 @@ private[jvm] object GeneratedClassHandler { } -/** Paths for a compilation unit, used during classfile writing */ -final case class CompilationUnitPaths(sourceFile: AbstractFile, outputDir: AbstractFile) { - def outputPath: Path = outputDir.file.toPath // `toPath` caches its result -} - /** * State for a compilation unit being post-processed. * - Holds the classes to post-process (released for GC when no longer used) * - Keeps a reference to the future that runs the post-processor * - Buffers messages reported during post-processing */ -final class CompilationUnitInPostProcess(private var classes: List[GeneratedClass], val paths: CompilationUnitPaths) { +final class CompilationUnitInPostProcess(private var classes: List[GeneratedClass], val sourceFile: AbstractFile) { def takeClasses(): List[GeneratedClass] = { val c = classes classes = Nil diff --git a/src/compiler/scala/tools/nsc/backend/jvm/PostProcessor.scala b/src/compiler/scala/tools/nsc/backend/jvm/PostProcessor.scala index c42a02c58439..52b39e40d204 100644 --- a/src/compiler/scala/tools/nsc/backend/jvm/PostProcessor.scala +++ b/src/compiler/scala/tools/nsc/backend/jvm/PostProcessor.scala @@ -58,7 +58,7 @@ abstract class PostProcessor extends PerRunInit { classfileWriter = classfileWriters.ClassfileWriter(global) } - def sendToDisk(clazz: GeneratedClass, paths: CompilationUnitPaths): Unit = { + def sendToDisk(clazz: GeneratedClass, sourceFile: AbstractFile): Unit = { val classNode = clazz.classNode val internalName = classNode.name val bytes = try { @@ -85,7 +85,7 @@ abstract class PostProcessor extends PerRunInit { if (AsmUtils.traceSerializedClassEnabled && internalName.contains(AsmUtils.traceSerializedClassPattern)) AsmUtils.traceClass(bytes) - classfileWriter.write(internalName, bytes, paths) + classfileWriter.writeClass(internalName, bytes, sourceFile) } } diff --git a/src/compiler/scala/tools/nsc/settings/ScalaSettings.scala b/src/compiler/scala/tools/nsc/settings/ScalaSettings.scala index 8b736448822d..1ef4b8c3120c 100644 --- a/src/compiler/scala/tools/nsc/settings/ScalaSettings.scala +++ b/src/compiler/scala/tools/nsc/settings/ScalaSettings.scala @@ -254,6 +254,7 @@ trait ScalaSettings extends AbsScalaSettings val YjarCompressionLevel = IntSetting("-Yjar-compression-level", "compression level to use when writing jar files", Deflater.DEFAULT_COMPRESSION, Some((Deflater.DEFAULT_COMPRESSION,Deflater.BEST_COMPRESSION)), (x: String) => None) val YpickleJava = BooleanSetting("-Ypickle-java", "Pickler phase should compute pickles for .java defined symbols for use by build tools").internalOnly() + val YpickleWrite = StringSetting("-Ypickle-write", "directory|jar", "destination for generated .sig files containing type signatures.", "", None).internalOnly() sealed abstract class CachePolicy(val name: String, val help: String) object CachePolicy { diff --git a/src/compiler/scala/tools/nsc/symtab/classfile/Pickler.scala b/src/compiler/scala/tools/nsc/symtab/classfile/Pickler.scala index 1fd7690763e5..b7fb20f590ca 100644 --- a/src/compiler/scala/tools/nsc/symtab/classfile/Pickler.scala +++ b/src/compiler/scala/tools/nsc/symtab/classfile/Pickler.scala @@ -16,6 +16,7 @@ package classfile import java.lang.Float.floatToIntBits import java.lang.Double.doubleToLongBits +import java.nio.file.Paths import scala.io.Codec import scala.reflect.internal.pickling.{PickleBuffer, PickleFormat} @@ -23,6 +24,7 @@ import scala.reflect.internal.util.shortClassOfInstance import scala.collection.mutable import PickleFormat._ import Flags._ +import scala.reflect.io.{AbstractFile, NoAbstractFile, PlainFile, PlainNioFile} /** * Serialize a top-level module and/or class. @@ -40,6 +42,13 @@ abstract class Pickler extends SubComponent { def newPhase(prev: Phase): StdPhase = new PicklePhase(prev) class PicklePhase(prev: Phase) extends StdPhase(prev) { + import global.genBCode.postProcessor.classfileWriters.FileWriter + private lazy val sigWriter: Option[FileWriter] = + if (settings.YpickleWrite.isSetByUser && !settings.YpickleWrite.value.isEmpty) + Some(FileWriter(global, new PlainFile(settings.YpickleWrite.value), None)) + else + None + def apply(unit: CompilationUnit): Unit = { def pickle(tree: Tree): Unit = { tree match { @@ -64,6 +73,7 @@ abstract class Pickler extends SubComponent { currentRun.symData(sym) = pickle } pickle.writeArray() + writeSigFile(sym, pickle) currentRun registerPickle sym } case _ => @@ -91,6 +101,27 @@ abstract class Pickler extends SubComponent { } } + override def run(): Unit = { + try super.run() + finally closeSigWriter() + } + + private def writeSigFile(sym: Symbol, pickle: PickleBuffer): Unit = { + sigWriter.foreach { writer => + val binaryName = sym.javaBinaryNameString + val binaryClassName = if (sym.isModule) binaryName.stripSuffix(nme.MODULE_SUFFIX_STRING) else binaryName + val relativePath = java.nio.file.Paths.get(binaryClassName + ".sig") + val data = pickle.bytes.take(pickle.writeIndex) + writer.writeFile(relativePath, data) + } + } + private def closeSigWriter(): Unit = { + sigWriter.foreach { writer => + writer.close() + reporter.info(NoPosition, "[sig files written]", force = false) + } + } + override protected def shouldSkipThisPhaseForJava: Boolean = !settings.YpickleJava.value } diff --git a/test/files/run/t5717.scala b/test/files/run/t5717.scala index 880d3c8e9128..c92ad650fdd8 100644 --- a/test/files/run/t5717.scala +++ b/test/files/run/t5717.scala @@ -20,9 +20,8 @@ object Test extends StoreReporterDirectTest { val List(i) = filteredInfos // for some reason, nio doesn't throw the same exception on windows and linux/mac val path = if(util.Properties.isWin)"\\a" else "/a" - val expected = "error writing a/B: Can't create directory " + path + + val expected = s"error writing ${testOutput.path}/a/B.class: Can't create directory ${testOutput.path}${path}" + "; there is an existing (non-directory) file in its path" - val actual = i.msg.replace(testOutput.path, "") - assert(actual == expected, actual) + assert(i.msg == expected, i.msg) } }