From 7b589dd4b2f3b310d6e4e26e76db3714ecfa0af2 Mon Sep 17 00:00:00 2001 From: Jason Zaugg Date: Wed, 19 Jun 2019 16:36:12 +1000 Subject: [PATCH 1/7] Backport changes to Names and ClassfileParser ------------------------------------------------------------------------ Reuse the buffer for classfile reading Classfile parsing does re-enter when we're reading package objects or classfiles for things like `scala/native.class`. But for the most part the prior refactorings mean that we typically only parse a single classfile at a time, and as such we can profit from a one-element cache for the buffer to read this into. (cherry picked from commit ed8d95eb3092a6fd239820362034b42ad636d85b) ------------------------------------------------------------------------ Eagerly read from the constant pool as a basis for lazy types java class/method I've used lazy types for field/method/class infos, which is analagous to what we do in `Unpickler` for scala originated types. We read all data needed by the inner class table and the type completers from the pool eagerly, but still be lazy about interning strings to Names and completion of the field/method types themselves. This fixes some long standing spurious cyclic errors: Manually tested with: ``` $ scalac -cp $(coursier fetch -q -p com.datastax.cassandra:dse-driver:1.0.0) test.scala test.scala:2: error: illegal cyclic reference involving class Cluster new com.datastax.driver.dse.DseCluster.Builder() ^ one error found $ /code/scala/build/quick/bin/scalac -cp $(coursier fetch -q -p com.datastax.cassandra:dse-driver:1.0.0) test.scala $ cat test.scala class Test { new com.datastax.driver.dse.DseCluster.Builder() } ``` ------------------------------------------------------------------------ Avoid using Names for fully qualified class names There is no good reason for these dotted names to be Names and stick around in the name table. Let's use short lived strings instead. Reduces the name table by 5% in terms of entries and 10% in terms of characters when compiling src/scalap/**/*.scala (cherry picked from commit ae18049a6c5f8851e01ac5baebb4b95262df0685) ------------------------------------------------------------------------ Avoid Names for descriptors, generic sigs, and string constants We can just keep these are short-lived Strings, rather than interning them into the Name table for the entire lifetime of Global. (cherry picked from commit 688bf0fcae4ced47fa440def73e3940005c841b1) ------------------------------------------------------------------------ Invalidate symbols for artifact classfiles, refactor classfile parser No longer run the classfile parser on Scala generated classfiles that don't have a Scala signature (module classes, inner classes, etc). Various cleanups in the classfile parser, minimize the work performed on Scala classfiles. Before, the attributes section was parsed twice: once to find the ScalaSig attribute, the second time to find the ScalaSignature in the RuntimeVisibleAnnotations. Now everything happens in the first iteration. Also fixes a bug in the backend: classes ending in `$` did not get a ScalaSignature by mistake. They were filtered out by the name-based test that is supposed to identify module classes. (cherry picked from commit 3aea776ca1aa82c9de44cc6806dcdb242f3b40f8) ------------------------------------------------------------------------ Remove unnecessary abstraction Added in ced7214959, no longer needed since ICodeReader is gone. (cherry picked from commit e216e0ef0376c550846de974d5b71b39b92120b8) --- src/compiler/scala/tools/nsc/Global.scala | 3 + .../tools/nsc/backend/jvm/BCodeHelpers.scala | 2 +- .../tools/nsc/symtab/SymbolLoaders.scala | 10 +- .../symtab/classfile/AbstractFileReader.scala | 32 +- .../symtab/classfile/ClassfileParser.scala | 977 ++++++++++-------- .../nsc/symtab/classfile/DataReader.scala | 68 ++ .../symtab/classfile/ReusableDataReader.scala | 156 +++ .../scala/reflect/internal/Definitions.scala | 25 +- .../scala/reflect/internal/Mirrors.scala | 73 +- .../scala/reflect/internal/Names.scala | 2 + .../scala/reflect/internal/StdNames.scala | 26 +- .../scala/reflect/internal/Symbols.scala | 11 +- .../scala/reflect/internal/Types.scala | 48 +- .../scala/reflect/io/AbstractFile.scala | 1 + src/reflect/scala/reflect/io/PlainFile.scala | 4 + .../reflect/runtime/JavaUniverseForce.scala | 1 - test/files/jvm/throws-annot-from-java.check | 10 +- .../jvm/throws-annot-from-java/Test_3.scala | 6 +- test/files/neg/moduleClassReference.check | 4 + test/files/neg/moduleClassReference.scala | 3 + test/files/neg/t7251.check | 2 +- test/files/run/compiler-asSeenFrom.scala | 2 +- test/files/run/existentials-in-compiler.scala | 4 +- .../t7008-scala-defined/Impls_Macros_2.scala | 2 + test/files/run/t7008/Impls_Macros_2.scala | 2 + test/files/run/t7096.scala | 2 +- test/files/run/t7455/Test.scala | 2 +- 27 files changed, 897 insertions(+), 581 deletions(-) create mode 100644 src/compiler/scala/tools/nsc/symtab/classfile/DataReader.scala create mode 100644 src/compiler/scala/tools/nsc/symtab/classfile/ReusableDataReader.scala create mode 100644 test/files/neg/moduleClassReference.check create mode 100644 test/files/neg/moduleClassReference.scala diff --git a/src/compiler/scala/tools/nsc/Global.scala b/src/compiler/scala/tools/nsc/Global.scala index eaaba1e99b2..9bf44d78976 100644 --- a/src/compiler/scala/tools/nsc/Global.scala +++ b/src/compiler/scala/tools/nsc/Global.scala @@ -1548,6 +1548,9 @@ class Global(var currentSettings: Settings, reporter0: Reporter) reporting.summarizeErrors() + // val allNamesArray: Array[String] = allNames().map(_.toString).toArray.sorted + // allNamesArray.foreach(println(_)) + if (traceSymbolActivity) units map (_.body) foreach (traceSymbols recordSymbolsInTree _) diff --git a/src/compiler/scala/tools/nsc/backend/jvm/BCodeHelpers.scala b/src/compiler/scala/tools/nsc/backend/jvm/BCodeHelpers.scala index 5fe51011b85..df9aa82a679 100644 --- a/src/compiler/scala/tools/nsc/backend/jvm/BCodeHelpers.scala +++ b/src/compiler/scala/tools/nsc/backend/jvm/BCodeHelpers.scala @@ -415,7 +415,7 @@ abstract class BCodeHelpers extends BCodeIdiomatic { */ def getAnnotPickle(jclassName: String, sym: Symbol): Option[AnnotationInfo] = { currentRun.symData get sym match { - case Some(pickle) if !sym.isModuleClass => + case Some(pickle) if !sym.isModuleClass => // pickles for module classes are in the companion / mirror class val scalaAnnot = { val sigBytes = ScalaSigBytes(pickle.bytes.take(pickle.writeIndex)) AnnotationInfo(sigBytes.sigAnnot, Nil, (nme.bytes, sigBytes) :: Nil) diff --git a/src/compiler/scala/tools/nsc/symtab/SymbolLoaders.scala b/src/compiler/scala/tools/nsc/symtab/SymbolLoaders.scala index 2ad68f4d620..847b1837bbe 100644 --- a/src/compiler/scala/tools/nsc/symtab/SymbolLoaders.scala +++ b/src/compiler/scala/tools/nsc/symtab/SymbolLoaders.scala @@ -13,13 +13,13 @@ package scala.tools.nsc package symtab -import classfile.ClassfileParser +import classfile.{ClassfileParser, ReusableDataReader} import java.io.IOException import scala.reflect.internal.MissingRequirementError import scala.reflect.io.{AbstractFile, NoAbstractFile} import scala.tools.nsc.util.{ClassPath, ClassRepresentation} import scala.reflect.internal.TypesStats -import scala.reflect.internal.util.StatisticsStatics +import scala.reflect.internal.util.{ReusableInstance, StatisticsStatics} /** This class ... * @@ -301,13 +301,11 @@ abstract class SymbolLoaders { } } } - + private val classFileDataReader: ReusableInstance[ReusableDataReader] = new ReusableInstance[ReusableDataReader](() => new ReusableDataReader()) class ClassfileLoader(val classfile: AbstractFile, clazz: ClassSymbol, module: ModuleSymbol) extends SymbolLoader with FlagAssigningCompleter { private object classfileParser extends { val symbolTable: SymbolLoaders.this.symbolTable.type = SymbolLoaders.this.symbolTable - } with ClassfileParser { - override protected type ThisConstantPool = ConstantPool - override protected def newConstantPool: ThisConstantPool = new ConstantPool + } with ClassfileParser(classFileDataReader) { override protected def lookupMemberAtTyperPhaseIfPossible(sym: Symbol, name: Name): Symbol = SymbolLoaders.this.lookupMemberAtTyperPhaseIfPossible(sym, name) /* diff --git a/src/compiler/scala/tools/nsc/symtab/classfile/AbstractFileReader.scala b/src/compiler/scala/tools/nsc/symtab/classfile/AbstractFileReader.scala index 19be00dd686..17d70998f3d 100644 --- a/src/compiler/scala/tools/nsc/symtab/classfile/AbstractFileReader.scala +++ b/src/compiler/scala/tools/nsc/symtab/classfile/AbstractFileReader.scala @@ -14,8 +14,10 @@ package scala.tools.nsc package symtab package classfile -import java.lang.Float.intBitsToFloat +import java.io.{ByteArrayInputStream, DataInputStream} import java.lang.Double.longBitsToDouble +import java.lang.Float.intBitsToFloat +import java.util import scala.tools.nsc.io.AbstractFile @@ -25,8 +27,11 @@ import scala.tools.nsc.io.AbstractFile * @author Philippe Altherr * @version 1.0, 23/03/2004 */ -class AbstractFileReader(val file: AbstractFile, val buf: Array[Byte]) { - def this(file: AbstractFile) = this(file, file.toByteArray) +final class AbstractFileReader(val buf: Array[Byte]) extends DataReader { + @deprecated("Use other constructor", "2.13.0") + def this(file: AbstractFile) { + this(file.toByteArray) + } /** the current input pointer */ @@ -59,17 +64,25 @@ class AbstractFileReader(val file: AbstractFile, val buf: Array[Byte]) { ((nextByte & 0xff) << 24) + ((nextByte & 0xff) << 16) + ((nextByte & 0xff) << 8) + (nextByte & 0xff) + /** extract a byte at position bp from buf + */ + def getByte(mybp: Int): Byte = + buf(mybp) + + def getBytes(mybp: Int, bytes: Array[Byte]): Unit = { + System.arraycopy(buf, mybp, bytes, 0, bytes.length) + } /** extract a character at position bp from buf */ def getChar(mybp: Int): Char = - (((buf(mybp) & 0xff) << 8) + (buf(mybp+1) & 0xff)).toChar + (((getByte(mybp) & 0xff) << 8) + (getByte(mybp+1) & 0xff)).toChar /** extract an integer at position bp from buf */ def getInt(mybp: Int): Int = - ((buf(mybp ) & 0xff) << 24) + ((buf(mybp+1) & 0xff) << 16) + - ((buf(mybp+2) & 0xff) << 8) + (buf(mybp+3) & 0xff) + ((getByte(mybp) & 0xff) << 24) + ((getByte(mybp + 1) & 0xff) << 16) + + ((getByte(mybp + 2) & 0xff) << 8) + (getByte(mybp + 3) & 0xff) /** extract a long integer at position bp from buf */ @@ -84,8 +97,11 @@ class AbstractFileReader(val file: AbstractFile, val buf: Array[Byte]) { */ def getDouble(mybp: Int): Double = longBitsToDouble(getLong(mybp)) + def getUTF(mybp: Int, len: Int): String = { + new DataInputStream(new ByteArrayInputStream(buf, mybp, len)).readUTF + } + /** skip next 'n' bytes */ - def skip(n: Int) { bp += n } - + def skip(n: Int): Unit = { bp += n } } diff --git a/src/compiler/scala/tools/nsc/symtab/classfile/ClassfileParser.scala b/src/compiler/scala/tools/nsc/symtab/classfile/ClassfileParser.scala index c855f1c11bb..f637f28d4ec 100644 --- a/src/compiler/scala/tools/nsc/symtab/classfile/ClassfileParser.scala +++ b/src/compiler/scala/tools/nsc/symtab/classfile/ClassfileParser.scala @@ -17,13 +17,15 @@ package classfile import java.io.{ByteArrayInputStream, DataInputStream, File, IOException} import java.lang.Integer.toHexString +import java.nio.ByteBuffer import scala.collection.{immutable, mutable} import scala.collection.mutable.{ArrayBuffer, ListBuffer} import scala.annotation.switch import scala.reflect.internal.JavaAccFlags -import scala.reflect.internal.pickling.{ByteCodecs, PickleBuffer} -import scala.reflect.io.NoAbstractFile +import scala.reflect.internal.pickling.ByteCodecs +import scala.reflect.internal.util.ReusableInstance +import scala.reflect.io.{NoAbstractFile, VirtualFile} import scala.reflect.internal.util.Collections._ import scala.tools.nsc.util.ClassPath import scala.tools.nsc.io.AbstractFile @@ -34,7 +36,7 @@ import scala.util.control.NonFatal * @author Martin Odersky * @version 1.0 */ -abstract class ClassfileParser { +abstract class ClassfileParser(reader: ReusableInstance[ReusableDataReader]) { val symbolTable: SymbolTable { def settings: Settings } @@ -60,21 +62,17 @@ abstract class ClassfileParser { import scala.reflect.internal.ClassfileConstants._ import Flags._ - protected type ThisConstantPool <: ConstantPool - protected def newConstantPool: ThisConstantPool - - protected var file: AbstractFile = _ // the class file - protected var in: AbstractFileReader = _ // the class file reader + protected var file: AbstractFile = _ // the class file + protected var in: DataReader = _ // the class file reader protected var clazz: ClassSymbol = _ // the class symbol containing dynamic members protected var staticModule: ModuleSymbol = _ // the module symbol containing static members protected var instanceScope: Scope = _ // the scope of all instance definitions protected var staticScope: Scope = _ // the scope of all static definitions - protected var pool: ThisConstantPool = _ // the classfile's constant pool + protected var pool: ConstantPool = _ // the classfile's constant pool protected var isScala: Boolean = _ // does class file describe a scala class? - protected var isScalaAnnot: Boolean = _ // does class file describe a scala class with its pickled info in an annotation? protected var isScalaRaw: Boolean = _ // this class file is a scala class with no pickled info protected var busy: Symbol = _ // lock to detect recursive reads - protected var currentClass: Name = _ // JVM name of the current class + protected var currentClass: String = _ // JVM name of the current class protected var classTParams = Map[Name,Symbol]() protected var srcfile0 : Option[AbstractFile] = None protected def moduleClass: Symbol = staticModule.moduleClass @@ -100,7 +98,7 @@ abstract class ClassfileParser { private def readMethodFlags() = JavaAccFlags methodFlags u2 private def readFieldFlags() = JavaAccFlags fieldFlags u2 private def readTypeName() = readName().toTypeName - private def readName() = pool getName u2 + private def readName() = pool.getName(u2).name private def readType() = pool getType u2 private object unpickler extends scala.reflect.internal.pickling.UnPickler { @@ -134,11 +132,6 @@ abstract class ClassfileParser { catch parseErrorHandler finally busy = NoSymbol } - @inline private def raiseLoaderLevel[T](body: => T): T = { - loaders.parentsLevel += 1 - try body - finally loaders.parentsLevel -= 1 - } /** * `clazz` and `module` are the class and module symbols corresponding to the classfile being @@ -152,20 +145,23 @@ abstract class ClassfileParser { def parse(file: AbstractFile, clazz: ClassSymbol, module: ModuleSymbol): Unit = { this.file = file pushBusy(clazz) { - this.clazz = clazz - this.staticModule = module - this.isScala = false - - this.in = new AbstractFileReader(file) - val magic = in.getInt(in.bp) - if (magic != JAVA_MAGIC && file.name.endsWith(".sig")) { - currentClass = TermName(clazz.javaClassName) - isScala = true - unpickler.unpickle(in.buf, 0, clazz, staticModule, file.name) - } else { - parseHeader() - this.pool = newConstantPool - parseClass() + reader.using { reader => + this.clazz = clazz + this.staticModule = module + this.isScala = false + + val fileContents = file.toByteArray + this.in = new AbstractFileReader(fileContents) + val magic = in.getInt(in.bp) + if (magic != JAVA_MAGIC && file.name.endsWith(".sig")) { + currentClass = clazz.javaClassName + isScala = true + unpickler.unpickle(fileContents, 0, clazz, staticModule, file.name) + } else { + parseHeader() + this.pool = new ConstantPool + parseClass() + } } } } @@ -173,11 +169,26 @@ abstract class ClassfileParser { private def parseHeader() { val magic = u4 if (magic != JAVA_MAGIC) - abort(s"class file ${in.file} has wrong magic number 0x${toHexString(magic)}") + abort(s"class file ${file} has wrong magic number 0x${toHexString(magic)}") val minor, major = u2 if (major < JAVA_MAJOR_VERSION || major == JAVA_MAJOR_VERSION && minor < JAVA_MINOR_VERSION) - abort(s"class file ${in.file} has unknown version $major.$minor, should be at least $JAVA_MAJOR_VERSION.$JAVA_MINOR_VERSION") + abort(s"class file ${file} has unknown version $major.$minor, should be at least $JAVA_MAJOR_VERSION.$JAVA_MINOR_VERSION") + } + + protected class NameOrString(val value: String) { + private var _name: Name = null + def name: Name = { + if (_name eq null) _name = TermName(value) + _name + } + } + + def getClassSymbol(name: String): Symbol = { + name match { + case name if name.endsWith(nme.MODULE_SUFFIX_STRING) => rootMirror getModuleByName newTermName(name).dropModule + case name => classNameToSymbol(name) + } } /** @@ -187,7 +198,9 @@ abstract class ClassfileParser { protected val len = u2 protected val starts = new Array[Int](len) protected val values = new Array[AnyRef](len) - protected val internalized = new Array[Name](len) + protected val internalized = new Array[NameOrString](len) + + val initBp = in.bp { var i = 1 while (i < starts.length) { @@ -205,7 +218,7 @@ abstract class ClassfileParser { } } } - + val endBp = in.bp def recordAtIndex[T <: AnyRef](value: T, idx: Int): T = { values(idx) = value value @@ -213,33 +226,30 @@ abstract class ClassfileParser { def firstExpecting(index: Int, expected: Int): Int = { val start = starts(index) - val first = in.buf(start).toInt + val first = in.getByte(start).toInt if (first == expected) start + 1 else this errorBadTag start } /** Return the name found at given index. */ - def getName(index: Int): Name = ( + def getName(index: Int): NameOrString = ( if (index <= 0 || len <= index) errorBadIndex(index) else values(index) match { - case name: Name => name + case name: NameOrString => name case _ => val start = firstExpecting(index, CONSTANT_UTF8) val len = in.getChar(start).toInt - recordAtIndex(TermName(fromMUTF8(in.buf, start, len + 2)), index) + recordAtIndex(new NameOrString(in.getUTF(start, len + 2)), index) } ) - private def fromMUTF8(bytes: Array[Byte], offset: Int, len: Int): String = - new DataInputStream(new ByteArrayInputStream(bytes, offset, len)).readUTF - /** Return the name found at given index in the constant pool, with '/' replaced by '.'. */ - def getExternalName(index: Int): Name = { + def getExternalName(index: Int): NameOrString = { if (index <= 0 || len <= index) errorBadIndex(index) if (internalized(index) == null) - internalized(index) = getName(index).replace('/', '.') + internalized(index) = new NameOrString(getName(index).value.replace('/', '.')) internalized(index) } @@ -249,10 +259,7 @@ abstract class ClassfileParser { values(index) match { case sym: Symbol => sym case _ => - val result = getClassName(index) match { - case name if nme.isModuleName(name) => rootMirror getModuleByName name.dropModule - case name => classNameToSymbol(name) - } + val result = ClassfileParser.this.getClassSymbol(getClassName(index).value) recordAtIndex(result, index) } } @@ -260,9 +267,9 @@ abstract class ClassfileParser { /** Return the external name of the class info structure found at 'index'. * Use 'getClassSymbol' if the class is sure to be a top-level class. */ - def getClassName(index: Int): Name = { + def getClassName(index: Int): NameOrString = { val start = firstExpecting(index, CONSTANT_CLASS) - getExternalName((in getChar start).toInt) + getExternalName((in.getChar(start)).toInt) } /** Return a name and a type at the given index. If the type is a method @@ -279,14 +286,14 @@ abstract class ClassfileParser { val start = firstExpecting(index, CONSTANT_NAMEANDTYPE) val name = getName(in.getChar(start).toInt) // create a dummy symbol for method types - val dummy = ownerTpe.typeSymbol.newMethod(name.toTermName, ownerTpe.typeSymbol.pos) + val dummy = ownerTpe.typeSymbol.newMethod(name.name.toTermName, ownerTpe.typeSymbol.pos) val tpe = getType(dummy, in.getChar(start + 2).toInt) // fix the return type, which is blindly set to the class currently parsed val restpe = tpe match { - case MethodType(formals, _) if name == nme.CONSTRUCTOR => MethodType(formals, ownerTpe) - case _ => tpe + case MethodType(formals, _) if name.name == nme.CONSTRUCTOR => MethodType(formals, ownerTpe) + case _ => tpe } - ((name, restpe)) + ((name.name, restpe)) } } @@ -301,21 +308,21 @@ abstract class ClassfileParser { case cls: Symbol => cls.tpe_* case _ => val name = getClassName(index) - name charAt 0 match { - case ARRAY_TAG => recordAtIndex(sigToType(null, name), index) - case _ => recordAtIndex(classNameToSymbol(name), index).tpe_* + name.value.charAt(0) match { + case ARRAY_TAG => recordAtIndex(sigToType(null, name.value), index) + case _ => recordAtIndex(classNameToSymbol(name.value), index).tpe_* } } } def getType(index: Int): Type = getType(null, index) - def getType(sym: Symbol, index: Int): Type = sigToType(sym, getExternalName(index)) - def getSuperClass(index: Int): Symbol = if (index == 0) AnyClass else getClassSymbol(index) // the only classfile that is allowed to have `0` in the super_class is java/lang/Object (see jvm spec) + def getType(sym: Symbol, index: Int): Type = sigToType(sym, getExternalName(index).value) + def getSuperClassName(index: Int): NameOrString = if (index == 0) null else getClassName(index) // the only classfile that is allowed to have `0` in the super_class is java/lang/Object (see jvm spec) private def createConstant(index: Int): Constant = { val start = starts(index) - Constant((in.buf(start).toInt: @switch) match { - case CONSTANT_STRING => getName(in.getChar(start + 1).toInt).toString + Constant((in.getByte(start).toInt: @switch) match { + case CONSTANT_STRING => getName(in.getChar(start + 1).toInt).value case CONSTANT_INTEGER => in.getInt(start + 1) case CONSTANT_FLOAT => in.getFloat(start + 1) case CONSTANT_LONG => in.getLong(start + 1) @@ -350,7 +357,7 @@ abstract class ClassfileParser { val start = firstExpecting(index, CONSTANT_UTF8) val len = (in getChar start).toInt val bytes = new Array[Byte](len) - System.arraycopy(in.buf, start + 2, bytes, 0, len) + in.getBytes(start + 2, bytes) recordAtIndex(getSubArray(bytes), index) } ) @@ -364,7 +371,10 @@ abstract class ClassfileParser { if (index <= 0 || ConstantPool.this.len <= index) errorBadIndex(index) val start = firstExpecting(index, CONSTANT_UTF8) val len = (in getChar start).toInt - in.buf drop start + 2 take len + val s = start + 2 + val result = new Array[Byte](len) + in.getBytes(s, result) + result } recordAtIndex(getSubArray(arr), head) } @@ -376,7 +386,7 @@ abstract class ClassfileParser { /** Throws an exception signaling a bad tag at given address. */ protected def errorBadTag(start: Int) = - abort(s"bad constant pool tag ${in.buf(start)} at byte $start") + abort(s"bad constant pool tag ${in.getByte(start)} at byte $start") } def stubClassSymbol(name: Name): Symbol = { @@ -392,13 +402,13 @@ abstract class ClassfileParser { NoSymbol.newStubSymbol(name.toTypeName, msg) } - private def lookupClass(name: Name) = try { + private def lookupClass(name: String) = try { def lookupTopLevel = { - if (name containsChar '.') + if (name contains '.') rootMirror getClassByName name else // FIXME - we shouldn't be doing ad hoc lookups in the empty package, getClassByName should return the class - definitions.getMember(rootMirror.EmptyPackageClass, name.toTypeName) + definitions.getMember(rootMirror.EmptyPackageClass, newTypeName(name)) } // For inner classes we usually don't get here: `classNameToSymbol` already returns the symbol @@ -409,21 +419,23 @@ abstract class ClassfileParser { // what the logic below is for (see PR #5822 / scala/bug#9937). val split = if (isScalaRaw) -1 else name.lastIndexOf('$') if (split > 0 && split < name.length) { - val outerName = name.subName(0, split) - val innerName = name.subName(split + 1, name.length).toTypeName + val outerName = name.substring(0, split) + val innerName = name.substring(split + 1, name.length) val outerSym = classNameToSymbol(outerName) // If the outer class C cannot be found, look for a top-level class C$D if (outerSym.isInstanceOf[StubSymbol]) lookupTopLevel else { + val innerNameAsName = newTypeName(innerName) + // We have a java-defined class name C$D and look for a member D of C. But we don't know if // D is declared static or not, so we have to search both in class C and its companion. val r = if (outerSym == clazz) - staticScope.lookup(innerName) orElse - instanceScope.lookup(innerName) + staticScope.lookup(innerNameAsName) orElse + instanceScope.lookup(innerNameAsName) else - lookupMemberAtTyperPhaseIfPossible(outerSym, innerName) orElse - lookupMemberAtTyperPhaseIfPossible(outerSym.companionModule, innerName) + lookupMemberAtTyperPhaseIfPossible(outerSym, innerNameAsName) orElse + lookupMemberAtTyperPhaseIfPossible(outerSym.companionModule, innerNameAsName) r orElse lookupTopLevel } } else @@ -434,14 +446,16 @@ abstract class ClassfileParser { // - was referenced in the bugfix commit for scala/bug#3756 (4fb0d53), not sure why // - covers the case when a type alias in a package object shadows a class symbol, // getClassByName throws a MissingRequirementError (scala-dev#248) - case _: FatalError => + case ex: FatalError => // getClassByName can throw a MissingRequirementError (which extends FatalError) // definitions.getMember can throw a FatalError, for example in pos/t5165b - stubClassSymbol(name) + if (settings.debug) + ex.printStackTrace() + stubClassSymbol(newTypeName(name)) } /** Return the class symbol of the given name. */ - def classNameToSymbol(name: Name): Symbol = { + def classNameToSymbol(name: String): Symbol = { if (innerClasses contains name) innerClasses innerSymbol name else @@ -449,87 +463,90 @@ abstract class ClassfileParser { } def parseClass() { - val jflags = readClassFlags() - val sflags = jflags.toScalaFlags - val nameIdx = u2 - currentClass = pool.getClassName(nameIdx) - - /* Parse parents for Java classes. For Scala, return AnyRef, since the real type will be unpickled. - * Updates the read pointer of 'in'. */ - def parseParents: List[Type] = { - if (isScala) { - u2 // skip superclass - val ifaces = u2 - in.bp += ifaces * 2 // .. and iface count interfaces - List(AnyRefTpe) // dummy superclass, will be replaced by pickled information - } - else raiseLoaderLevel { - val superType = if (jflags.isAnnotation) { u2; AnnotationClass.tpe } - else pool.getSuperClass(u2).tpe_* - val ifaceCount = u2 - var ifaces = for (i <- List.range(0, ifaceCount)) yield pool.getSuperClass(u2).tpe_* - if (jflags.isAnnotation) ifaces ::= ClassfileAnnotationClass.tpe - superType :: ifaces - } - } + unpickleOrParseInnerClasses() - val isTopLevel = !(currentClass containsChar '$') // Java class name; *don't* try to to use Scala name decoding (scala/bug#7532) + val jflags = readClassFlags() + val classNameIndex = u2 + currentClass = pool.getClassName(classNameIndex).value + + // Ensure that (top-level) classfiles are in the correct directory + val isTopLevel = !(currentClass contains '$') // Java class name; *don't* try to to use Scala name decoding (scala/bug#7532) if (isTopLevel) { - val c = pool.getClassSymbol(nameIdx) + val c = pool.getClassSymbol(classNameIndex) // scala-dev#248: when a type alias (in a package object) shadows a class symbol, getClassSymbol returns a stub + // TODO: this also prevents the error when it would be useful (`mv a/C.class .`) if (!c.isInstanceOf[StubSymbol] && c != clazz) mismatchError(c) } - addEnclosingTParams(clazz) - parseInnerClasses() // also sets the isScala / isScalaRaw flags, see r15956 - // get the class file parser to reuse scopes. - instanceScope = newScope - staticScope = newScope + // TODO: remove after the next 2.13 milestone + // A bug in the backend caused classes ending in `$` do get only a Scala marker attribute + // instead of a ScalaSig and a Signature annotaiton. This went unnoticed because isScalaRaw + // classes were parsed like Java classes. The below covers the cases in the std lib. + def isNothingOrNull = { + val n = clazz.fullName.toString + n == "scala.runtime.Nothing$" || n == "scala.runtime.Null$" + } + + if (isScala) { + () // We're done + } else if (isScalaRaw && !isNothingOrNull) { + val decls = clazz.enclosingPackage.info.decls + for (c <- List(clazz, staticModule, staticModule.moduleClass)) { + c.setInfo(NoType) + decls.unlink(c) + } + } else { + val sflags = jflags.toScalaFlags // includes JAVA + + addEnclosingTParams(clazz) - val classInfo = ClassInfoType(parseParents, instanceScope, clazz) - val staticInfo = ClassInfoType(List(), staticScope, moduleClass) + // Create scopes before calling `enterOwnInnerClasses` + instanceScope = newScope + staticScope = newScope + val staticInfo = ClassInfoType(List(), staticScope, moduleClass) + + val parentIndex = u2 + val parentName = if (parentIndex == 0) null else pool.getClassName(parentIndex) + val ifaceCount = u2 + val ifaces = for (i <- List.range(0, ifaceCount)) yield pool.getSuperClassName(u2) + val completer = new ClassTypeCompleter(clazz.name, jflags, parentName, ifaces) - if (!isScala && !isScalaRaw) enterOwnInnerClasses() - val curbp = in.bp - skipMembers() // fields - skipMembers() // methods - if (!isScala) { + clazz setInfo completer clazz setFlag sflags - propagatePackageBoundary(jflags, clazz, staticModule, staticModule.moduleClass) - clazz setInfo classInfo moduleClass setInfo staticInfo + moduleClass setFlag JAVA staticModule setInfo moduleClass.tpe staticModule setFlag JAVA - staticModule.moduleClass setFlag JAVA - // attributes now depend on having infos set already - parseAttributes(clazz, classInfo) - - def queueLoad() { - in.bp = curbp - 0 until u2 foreach (_ => parseField()) - sawPrivateConstructor = false - 0 until u2 foreach (_ => parseMethod()) - val needsConstructor = ( - !sawPrivateConstructor - && !(instanceScope containsName nme.CONSTRUCTOR) - && (sflags & INTERFACE) == 0 - ) - if (needsConstructor) - instanceScope enter clazz.newClassConstructor(NoPosition) - } - loaders.pendingLoadActions ::= (queueLoad _) - if (loaders.parentsLevel == 0) { - while (loaders.pendingLoadActions.nonEmpty) { - val item = loaders.pendingLoadActions.head - loaders.pendingLoadActions = loaders.pendingLoadActions.tail - item() - } - } - } else - parseAttributes(clazz, classInfo) + propagatePackageBoundary(jflags, clazz, staticModule, moduleClass) + + val fieldsStartBp = in.bp + skipMembers() // fields + skipMembers() // methods + + parseAttributes(clazz, completer) + + in.bp = fieldsStartBp + 0 until u2 foreach (_ => parseField()) + sawPrivateConstructor = false + 0 until u2 foreach (_ => parseMethod()) + val needsConstructor = ( + !sawPrivateConstructor + && !(instanceScope containsName nme.CONSTRUCTOR) + && ((sflags & INTERFACE) == 0) + ) + if (needsConstructor) + instanceScope enter clazz.newClassConstructor(NoPosition) + + // we could avoid this if we eagerly created class type param symbols here to expose through the + // ClassTypeCompleter to satisfy the calls to rawInfo.typeParams from Symbol.typeParams. That would + // require a refactor of `sigToType`. + // + // We would also need to make sure that clazzTParams is populated before member type completers called sig2type. + clazz.initialize + } } /** Add type parameters of enclosing classes */ @@ -551,17 +568,17 @@ abstract class ClassfileParser { in.skip(4); skipAttributes() } else { val name = readName() - val info = readType() + val lazyInfo = new MemberTypeCompleter(name, jflags, pool.getExternalName(u2).value) val sym = ownerForFlags(jflags).newValue(name.toTermName, NoPosition, sflags) // Note: the info may be overwritten later with a generic signature // parsed from SignatureATTR sym setInfo { if (jflags.isEnum) ConstantType(Constant(sym)) - else info + else lazyInfo } propagatePackageBoundary(jflags, sym) - parseAttributes(sym, info) + parseAttributes(sym, lazyInfo) addJavaFlagsAnnotations(sym, jflags) getScope(jflags) enter sym @@ -586,8 +603,8 @@ abstract class ClassfileParser { val jflags = readMethodFlags() val sflags = jflags.toScalaFlags if (jflags.isPrivate) { - val name = readName() - if (name == nme.CONSTRUCTOR) + val isConstructor = pool.getName(u2).value == "" // opt avoid interning a Name for private methods we're about to discard + if (isConstructor) sawPrivateConstructor = true in.skip(2); skipAttributes() } else { @@ -596,63 +613,30 @@ abstract class ClassfileParser { } else { val name = readName() val sym = ownerForFlags(jflags).newMethod(name.toTermName, NoPosition, sflags) - var info = pool.getType(sym, u2) - var removedOuterParameter = false - if (name == nme.CONSTRUCTOR) - info match { - case MethodType(params, restpe) => - // if this is a non-static inner class, remove the explicit outer parameter - val paramsNoOuter = innerClasses getEntry currentClass match { - case Some(entry) if !isScalaRaw && !entry.jflags.isStatic => - /* About `clazz.owner.hasPackageFlag` below: scala/bug#5957 - * For every nested java class A$B, there are two symbols in the scala compiler. - * 1. created by SymbolLoader, because of the existence of the A$B.class file, owner: package - * 2. created by ClassfileParser of A when reading the inner classes, owner: A - * If symbol 1 gets completed (e.g. because the compiled source mentions `A$B`, not `A#B`), the - * ClassfileParser for 1 executes, and clazz.owner is the package. - */ - assert(params.head.tpe.typeSymbol == clazz.owner || clazz.owner.hasPackageFlag, params.head.tpe.typeSymbol + ": " + clazz.owner) - removedOuterParameter = true - params.tail - case _ => - params - } - val newParams = paramsNoOuter match { - case (init :+ tail) if jflags.isSynthetic => - // scala/bug#7455 strip trailing dummy argument ("access constructor tag") from synthetic constructors which - // are added when an inner class needs to access a private constructor. - init - case _ => - paramsNoOuter - } - - info = MethodType(newParams, clazz.tpe) - } // Note: the info may be overwritten later with a generic signature // parsed from SignatureATTR - sym setInfo info + val lazyInfo = new MemberTypeCompleter(name, jflags, pool.getExternalName(u2).value) + sym.info = lazyInfo propagatePackageBoundary(jflags, sym) - parseAttributes(sym, info, removedOuterParameter) + parseAttributes(sym, lazyInfo) addJavaFlagsAnnotations(sym, jflags) - if (jflags.isVarargs) - sym modifyInfo arrayToRepeated - getScope(jflags) enter sym } } } - private def sigToType(sym: Symbol, sig: Name): Type = { + private def sigToType(sym: Symbol, sig: String): Type = { + val sigChars = sig.toCharArray var index = 0 val end = sig.length def accept(ch: Char) { assert(sig.charAt(index) == ch, (sig.charAt(index), ch)) index += 1 } - def subName(isDelimiter: Char => Boolean): Name = { + def subName(isDelimiter: Char => Boolean): String = { val start = index while (!isDelimiter(sig.charAt(index))) { index += 1 } - sig.subName(start, index) + new String(sigChars, start, index - start) } def sig2type(tparams: immutable.Map[Name,Symbol], skiptvs: Boolean): Type = { val tag = sig.charAt(index); index += 1 @@ -724,7 +708,7 @@ abstract class ClassfileParser { var tpe = processClassType(processInner(classSym.tpe_*)) while (sig.charAt(index) == '.') { accept('.') - val name = subName(c => c == ';' || c == '<' || c == '.').toTypeName + val name = newTypeName(subName(c => c == ';' || c == '<' || c == '.')) val clazz = tpe.member(name) val dummyArgs = Nil // the actual arguments are added in processClassType val inner = typeRef(pre = tpe, sym = clazz, args = dummyArgs) @@ -761,7 +745,7 @@ abstract class ClassfileParser { sig2type(tparams, skiptvs) JavaMethodType(sym.newSyntheticValueParams(paramtypes.toList), restype) case 'T' => - val n = subName(';'.==).toTypeName + val n = newTypeName(subName(';'.==)) index += 1 if (skiptvs) AnyTpe else tparams(n).typeConstructor @@ -785,7 +769,7 @@ abstract class ClassfileParser { index += 1 val start = index while (sig.charAt(index) != '>') { - val tpname = subName(':'.==).toTypeName + val tpname = newTypeName(subName(':'.==)) val s = sym.newTypeParameter(tpname) tparams = tparams + (tpname -> s) sig2typeBounds(tparams, skiptvs = true) @@ -793,7 +777,7 @@ abstract class ClassfileParser { } index = start while (sig.charAt(index) != '>') { - val tpname = subName(':'.==).toTypeName + val tpname = newTypeName(subName(':'.==)) val s = tparams(tpname) s.setInfo(sig2typeBounds(tparams, skiptvs = false)) } @@ -816,96 +800,58 @@ abstract class ClassfileParser { GenPolyType(ownTypeParams, tpe) } // sigToType - def parseAttributes(sym: Symbol, symtype: Type, removedOuterParameter: Boolean = false) { - var paramNames: ListBuffer[Name] = null // null means we didn't find any - def convertTo(c: Constant, pt: Type): Constant = { - if (pt.typeSymbol == BooleanClass && c.tag == IntTag) - Constant(c.value != 0) - else - c convertTo pt - } - def parseAttribute() { + /** + * Only invoked for java classfiles. + */ + private def parseAttributes(sym: symbolTable.Symbol, completer: JavaTypeCompleter): Unit = { + def parseAttribute(): Unit = { val attrName = readTypeName() val attrLen = u4 attrName match { case tpnme.SignatureATTR => - if (!isScala && !isScalaRaw) { - val sig = pool.getExternalName(u2) - val newType = sigToType(sym, sig) - sym.setInfo(newType) - } - else in.skip(attrLen) + val sigIndex = u2 + val sig = pool.getExternalName(sigIndex) + assert(sym.rawInfo == completer, sym) + completer.sig = sig.value case tpnme.SyntheticATTR => sym.setFlag(SYNTHETIC | ARTIFACT) in.skip(attrLen) + case tpnme.BridgeATTR => sym.setFlag(BRIDGE | ARTIFACT) in.skip(attrLen) + case tpnme.DeprecatedATTR => val arg = Literal(Constant("see corresponding Javadoc for more information.")) sym.addAnnotation(DeprecatedAttr, arg, Literal(Constant(""))) in.skip(attrLen) + case tpnme.ConstantValueATTR => - val c = pool.getConstant(u2) - val c1 = convertTo(c, symtype) - if (c1 ne null) sym.setInfo(ConstantType(c1)) - else devWarning(s"failure to convert $c to $symtype") + completer.constant = pool.getConstant(u2) + case tpnme.MethodParametersATTR => def readParamNames(): Unit = { - import scala.tools.asm.Opcodes.ACC_SYNTHETIC val paramCount = u1 + val paramNames = new Array[NameOrString](paramCount) + val paramNameAccess = new Array[Int](paramCount) var i = 0 - if (removedOuterParameter && i < paramCount) { - in.skip(4) - i += 1 - } - paramNames = new ListBuffer() while (i < paramCount) { - val rawname = pool.getName(u2) - val access = u2 - - val name = - if ((access & ACC_SYNTHETIC) == 0) rawname.encode - else nme.NO_NAME - - paramNames += name + paramNames(i) = pool.getExternalName(u2) + paramNameAccess(i) = u2 i += 1 } + completer.paramNames = new ParamNames(paramNames, paramNameAccess) } readParamNames() - case tpnme.ScalaSignatureATTR => - if (!isScalaAnnot) { - devWarning(s"symbol ${sym.fullName} has pickled signature in attribute") - unpickler.unpickle(in.buf, in.bp, clazz, staticModule, in.file.name) - } - in.skip(attrLen) - case tpnme.ScalaATTR => - isScalaRaw = true - // Attribute on methods of java annotation classes when that method has a default - case tpnme.AnnotationDefaultATTR => + + case tpnme.AnnotationDefaultATTR => // Methods of java annotation classes that have a default sym.addAnnotation(AnnotationDefaultAttr) in.skip(attrLen) - // Java annotations on classes / methods / fields with RetentionPolicy.RUNTIME + case tpnme.RuntimeAnnotationATTR => - if (isScalaAnnot || !isScala) { - // For Scala classfiles we are only interested in the scala signature annotations. Other - // annotations should be skipped (the pickle contains the symbol's annotations). - // Skipping them also prevents some spurious warnings / errors related to scala/bug#7014, - // scala/bug#7551, pos/5165b - val scalaSigAnnot = parseAnnotations(onlyScalaSig = isScalaAnnot) - if (isScalaAnnot) scalaSigAnnot match { - case Some(san: AnnotationInfo) => - val bytes = - san.assocs.find({ _._1 == nme.bytes }).get._2.asInstanceOf[ScalaSigBytes].bytes - - unpickler.unpickle(bytes, 0, clazz, staticModule, in.file.name) - case None => - throw new RuntimeException("Scala class file does not contain Scala annotation") - } - debuglog("[class] << " + sym.fullName + sym.annotationsString) - } - else - in.skip(attrLen) + val numAnnots = u2 + for (n <- 0 until numAnnots; annot <- parseAnnotation(u2)) + sym.addAnnotation(annot) // TODO 1: parse runtime visible annotations on parameters // case tpnme.RuntimeParamAnnotationATTR @@ -913,8 +859,8 @@ abstract class ClassfileParser { // TODO 2: also parse RuntimeInvisibleAnnotation / RuntimeInvisibleParamAnnotation, // i.e. java annotations with RetentionPolicy.CLASS? - case tpnme.ExceptionsATTR if (!isScala) => - parseExceptions(attrLen) + case tpnme.ExceptionsATTR => + parseExceptions(attrLen, completer) case tpnme.SourceFileATTR => if (forInteractive) { @@ -935,196 +881,108 @@ abstract class ClassfileParser { case rootMirror.EmptyPackage => srcfileLeaf case pkg => pkg.fullName(File.separatorChar)+File.separator+srcfileLeaf } - srcfile0 = settings.outputDirs.srcFilesFor(in.file, srcpath).find(_.exists) + srcfile0 = settings.outputDirs.srcFilesFor(file, srcpath).find(_.exists) } else in.skip(attrLen) + case tpnme.CodeATTR => if (sym.owner.isInterface) { sym setFlag JAVA_DEFAULTMETHOD log(s"$sym in ${sym.owner} is a java8+ default method.") } in.skip(attrLen) + case _ => in.skip(attrLen) } } - def skipAnnotArg(): Unit = { - u1 match { - case STRING_TAG | BOOL_TAG | BYTE_TAG | CHAR_TAG | SHORT_TAG | - INT_TAG | LONG_TAG | FLOAT_TAG | DOUBLE_TAG | CLASS_TAG => - in.skip(2) - - case ENUM_TAG => - in.skip(4) - - case ARRAY_TAG => - val num = u2 - for (i <- 0 until num) skipAnnotArg() - - case ANNOTATION_TAG => - parseAnnotation(u2, onlyScalaSig = true) - } - } - - def parseAnnotArg: Option[ClassfileAnnotArg] = { - val tag = u1 - val index = u2 - tag match { - case STRING_TAG => - Some(LiteralAnnotArg(Constant(pool.getName(index).toString))) - case BOOL_TAG | BYTE_TAG | CHAR_TAG | SHORT_TAG | INT_TAG | - LONG_TAG | FLOAT_TAG | DOUBLE_TAG => - Some(LiteralAnnotArg(pool.getConstant(index))) - case CLASS_TAG => - Some(LiteralAnnotArg(Constant(pool.getType(index)))) - case ENUM_TAG => - val t = pool.getType(index) - val n = readName() - val module = t.typeSymbol.companionModule - val s = module.info.decls.lookup(n) - if (s != NoSymbol) Some(LiteralAnnotArg(Constant(s))) - else { - warning( - sm"""While parsing annotations in ${in.file}, could not find $n in enum ${module.nameString}. - |This is likely due to an implementation restriction: an annotation argument cannot refer to a member of the annotated class (scala/bug#7014).""" - ) - None - } - - case ARRAY_TAG => - val arr = new ArrayBuffer[ClassfileAnnotArg]() - var hasError = false - for (i <- 0 until index) - parseAnnotArg match { - case Some(c) => arr += c - case None => hasError = true - } - if (hasError) None - else Some(ArrayAnnotArg(arr.toArray)) - case ANNOTATION_TAG => - parseAnnotation(index, onlyScalaSig = false) map (NestedAnnotArg(_)) - } - } - - def parseScalaSigBytes: Option[ScalaSigBytes] = { - val tag = u1 - assert(tag == STRING_TAG, tag) - Some(ScalaSigBytes(pool getBytes u2)) - } - - def parseScalaLongSigBytes: Option[ScalaSigBytes] = { - val tag = u1 - assert(tag == ARRAY_TAG, tag) - val stringCount = u2 - val entries = - for (i <- 0 until stringCount) yield { - val stag = u1 - assert(stag == STRING_TAG, stag) - u2 - } - Some(ScalaSigBytes(pool.getBytes(entries.toList))) - } - - // TODO scala/bug#9296 duplicated code, refactor - /* Parse and return a single annotation. If it is malformed, - * return None. - */ - def parseAnnotation(attrNameIndex: Int, onlyScalaSig: Boolean): Option[AnnotationInfo] = try { - val attrType = pool.getType(attrNameIndex) - val nargs = u2 - val nvpairs = new ListBuffer[(Name, ClassfileAnnotArg)] - var hasError = false - for (i <- 0 until nargs) { - val name = readName() - // The "bytes: String" argument of the ScalaSignature attribute is parsed specially so that it is - // available as an array of bytes (the pickled Scala signature) instead of as a string. The pickled signature - // is encoded as a string because of limitations in the Java class file format. - if ((attrType == ScalaSignatureAnnotation.tpe) && (name == nme.bytes)) - parseScalaSigBytes match { - case Some(c) => nvpairs += ((name, c)) - case None => hasError = true - } - else if ((attrType == ScalaLongSignatureAnnotation.tpe) && (name == nme.bytes)) - parseScalaLongSigBytes match { - case Some(c) => nvpairs += ((name, c)) - case None => hasError = true - } - else - if (onlyScalaSig) skipAnnotArg() - else parseAnnotArg match { - case Some(c) => nvpairs += ((name, c)) - case None => hasError = true - } - } - if (hasError) None - else Some(AnnotationInfo(attrType, List(), nvpairs.toList)) - } catch { - case f: FatalError => throw f // don't eat fatal errors, they mean a class was not found - case NonFatal(ex) => - // We want to be robust when annotations are unavailable, so the very least - // we can do is warn the user about the exception - // There was a reference to ticket 1135, but that is outdated: a reference to a class not on - // the classpath would *not* end up here. A class not found is signaled - // with a `FatalError` exception, handled above. Here you'd end up after a NPE (for example), - // and that should never be swallowed silently. - warning(s"Caught: $ex while parsing annotations in ${in.file}") - if (settings.debug) ex.printStackTrace() - None // ignore malformed annotations - } - /* * Parse the "Exceptions" attribute which denotes the exceptions * thrown by a method. */ - def parseExceptions(len: Int) { + def parseExceptions(len: Int, completer: JavaTypeCompleter): Unit = { val nClasses = u2 for (n <- 0 until nClasses) { // FIXME: this performs an equivalent of getExceptionTypes instead of getGenericExceptionTypes (scala/bug#7065) - val cls = pool.getClassSymbol(u2) - // we call initialize due to the fact that we call Symbol.isMonomorphicType in addThrowsAnnotation - // and that method requires Symbol to be forced to give the right answers, see scala/bug#7107 for details - cls.initialize - sym.addThrowsAnnotation(cls) + val cls = pool.getClassName(u2) + completer.exceptions ::= cls } } + // begin parseAttributes + for (i <- 0 until u2) parseAttribute() + } - /* Parse a sequence of annotations and attaches them to the - * current symbol sym, except for the ScalaSignature annotation that it returns, if it is available. */ - def parseAnnotations(onlyScalaSig: Boolean): Option[AnnotationInfo] = { - val nAttr = u2 - var scalaSigAnnot: Option[AnnotationInfo] = None - for (n <- 0 until nAttr) parseAnnotation(u2, onlyScalaSig) match { - case Some(scalaSig) if scalaSig.atp == ScalaSignatureAnnotation.tpe => - scalaSigAnnot = Some(scalaSig) - case Some(scalaSig) if scalaSig.atp == ScalaLongSignatureAnnotation.tpe => - scalaSigAnnot = Some(scalaSig) - case Some(annot) => - sym.addAnnotation(annot) - case None => - } - scalaSigAnnot + def parseAnnotArg(): Option[ClassfileAnnotArg] = { + val tag = u1 + val index = u2 + tag match { + case STRING_TAG => + Some(LiteralAnnotArg(Constant(pool.getName(index).value))) + case BOOL_TAG | BYTE_TAG | CHAR_TAG | SHORT_TAG | INT_TAG | + LONG_TAG | FLOAT_TAG | DOUBLE_TAG => + Some(LiteralAnnotArg(pool.getConstant(index))) + case CLASS_TAG => + Some(LiteralAnnotArg(Constant(pool.getType(index)))) + case ENUM_TAG => + val t = pool.getType(index) + val n = readName() + val module = t.typeSymbol.companionModule + val s = module.info.decls.lookup(n) + if (s != NoSymbol) Some(LiteralAnnotArg(Constant(s))) + else { + warning( + sm"""While parsing annotations in ${file}, could not find $n in enum ${module.nameString}. + |This is likely due to an implementation restriction: an annotation argument cannot refer to a member of the annotated class (scala/bug#7014).""" + ) + None + } + + case ARRAY_TAG => + val arr = new ArrayBuffer[ClassfileAnnotArg]() + var hasError = false + for (i <- 0 until index) + parseAnnotArg() match { + case Some(c) => arr += c + case None => hasError = true + } + if (hasError) None + else Some(ArrayAnnotArg(arr.toArray)) + case ANNOTATION_TAG => + parseAnnotation(index) map (NestedAnnotArg(_)) } + } - def addParamNames(): Unit = - if ((paramNames ne null) && sym.hasRawInfo && sym.isMethod) { - val params = sym.rawInfo.params - foreach2(paramNames.toList, params) { - case (nme.NO_NAME, _) => // param was ACC_SYNTHETIC; ignore - case (name, param) => - param.resetFlag(SYNTHETIC) - param.name = name - } - devWarningIf(!sameLength(paramNames.toList, params)) { - // there's not anything we can do, but it's slightly worrisome - sm"""MethodParameters length mismatch while parsing $sym: - | rawInfo.params: ${sym.rawInfo.params} - | MethodParameters: ${paramNames.toList}""" - } - } - // begin parseAttributes - for (i <- 0 until u2) parseAttribute() - addParamNames() + // TODO scala/bug#9296 duplicated code, refactor + /** + * Parse and return a single annotation. If it is malformed, return None. + */ + def parseAnnotation(attrNameIndex: Int): Option[AnnotationInfo] = try { + val attrType = pool.getType(attrNameIndex) + val nargs = u2 + val nvpairs = new ListBuffer[(Name, ClassfileAnnotArg)] + var hasError = false + for (i <- 0 until nargs) { + val name = readName() + parseAnnotArg() match { + case Some(c) => nvpairs += ((name, c)) + case None => hasError = true + } + } + if (hasError) None + else Some(AnnotationInfo(attrType, List(), nvpairs.toList)) + } catch { + case f: FatalError => throw f // don't eat fatal errors, they mean a class was not found + case NonFatal(ex) => + // We want to be robust when annotations are unavailable, so the very least + // we can do is warn the user about the exception + // There was a reference to ticket 1135, but that is outdated: a reference to a class not on + // the classpath would *not* end up here. A class not found is signaled + // with a `FatalError` exception, handled above. Here you'd end up after a NPE (for example), + // and that should never be swallowed silently. + warning(s"Caught: $ex while parsing annotations in ${file}") + if (settings.debug) ex.printStackTrace() + None // ignore malformed annotations } /** Apply `@native`/`@transient`/`@volatile` annotations to `sym`, @@ -1136,9 +994,9 @@ abstract class ClassfileParser { /** Enter own inner classes in the right scope. It needs the scopes to be set up, * and implicitly current class' superclasses. */ - private def enterOwnInnerClasses() { - def className(name: Name): Name = - name.subName(name.lastPos('.') + 1, name.length) + private def enterOwnInnerClasses(): Unit = { + def className(name: String): String = + name.substring(name.lastIndexOf('.') + 1, name.length) def enterClassAndModule(entry: InnerClassEntry, file: AbstractFile) { def jflags = entry.jflags @@ -1186,8 +1044,8 @@ abstract class ClassfileParser { decls unlink e } - val cName = className(entry.externalName) - unlinkIfPresent(cName.toTermName) + val cName = newTermName(className(entry.externalName)) + unlinkIfPresent(cName) unlinkIfPresent(cName.toTypeName) } @@ -1200,54 +1058,145 @@ abstract class ClassfileParser { } } - /** Parse inner classes. Expects `in.bp` to point to the superclass entry. - * Restores the old `bp`. + /** + * Either + * - set `isScala` and invoke the unpickler, or + * - set `isScalaRaw`, or + * - parse inner classes (for Java classfiles) + * + * Expects `in.bp` to point to the `access_flags` entry, restores the old `bp`. */ - def parseInnerClasses() { + def unpickleOrParseInnerClasses() { val oldbp = in.bp + in.skip(4) // access_flags, this_class skipSuperclasses() skipMembers() // fields skipMembers() // methods - val attrs = u2 - for (i <- 0 until attrs) { + + var innersStart = -1 + var runtimeAnnotStart = -1 + + val numAttrs = u2 + var i = 0 + while (i < numAttrs) { val attrName = readTypeName() val attrLen = u4 attrName match { case tpnme.ScalaSignatureATTR => isScala = true - val pbuf = new PickleBuffer(in.buf, in.bp, in.bp + attrLen) - pbuf.readNat(); pbuf.readNat() - if (pbuf.readNat == 0) // a scala signature attribute with no entries means that the actual scala signature - isScalaAnnot = true // is in a ScalaSignature annotation. - in.skip(attrLen) + if (runtimeAnnotStart != -1) i = numAttrs case tpnme.ScalaATTR => isScalaRaw = true - case tpnme.InnerClassesATTR if !isScala => - val entries = u2 - for (i <- 0 until entries) { - val innerIndex, outerIndex, nameIndex = u2 - val jflags = readInnerClassFlags() - if (innerIndex != 0 && outerIndex != 0 && nameIndex != 0) - innerClasses add InnerClassEntry(innerIndex, outerIndex, nameIndex, jflags) + i = numAttrs + case tpnme.InnerClassesATTR => + innersStart = in.bp + case tpnme.RuntimeAnnotationATTR => + runtimeAnnotStart = in.bp + if (isScala) i = numAttrs + case _ => + } + in.skip(attrLen) + i += 1 + } + + if (isScala) { + def parseScalaSigBytes(): Array[Byte] = { + val tag = u1 + assert(tag == STRING_TAG, tag) + pool.getBytes(u2) + } + + def parseScalaLongSigBytes(): Array[Byte] = { + val tag = u1 + assert(tag == ARRAY_TAG, tag) + val stringCount = u2 + val entries = + for (i <- 0 until stringCount) yield { + val stag = u1 + assert(stag == STRING_TAG, stag) + u2 } + pool.getBytes(entries.toList) + } + + def checkScalaSigAnnotArg() = { + val numArgs = u2 + assert(numArgs == 1, s"ScalaSignature has $numArgs arguments") + val name = readName() + assert(name == nme.bytes, s"ScalaSignature argument has name $name") + } + + def skipAnnotArg(): Unit = u1 match { + case STRING_TAG | BOOL_TAG | BYTE_TAG | CHAR_TAG | SHORT_TAG | + INT_TAG | LONG_TAG | FLOAT_TAG | DOUBLE_TAG | CLASS_TAG => + in.skip(2) + + case ENUM_TAG => + in.skip(4) + + case ARRAY_TAG => + val num = u2 + for (i <- 0 until num) skipAnnotArg() + + case ANNOTATION_TAG => + in.skip(2) // type + skipAnnotArgs() + } + + def skipAnnotArgs() = { + val numArgs = u2 + for (i <- 0 until numArgs) { + in.skip(2) + skipAnnotArg() + } + } + + val SigTpe = ScalaSignatureAnnotation.tpe + val LongSigTpe = ScalaLongSignatureAnnotation.tpe + + assert(runtimeAnnotStart != -1, s"No RuntimeVisibleAnnotations in classfile with ScalaSignature attribute: $clazz") + in.bp = runtimeAnnotStart + val numAnnots = u2 + var i = 0 + var bytes: Array[Byte] = null + while (i < numAnnots && bytes == null) pool.getType(u2) match { + case SigTpe => + checkScalaSigAnnotArg() + bytes = parseScalaSigBytes() + case LongSigTpe => + checkScalaSigAnnotArg() + bytes = parseScalaLongSigBytes() case _ => - in.skip(attrLen) + skipAnnotArgs() + } + + AnyRefClass // Force scala.AnyRef, otherwise we get "error: Symbol AnyRef is missing from the classpath" + assert(bytes != null, s"No Scala(Long)Signature annotation in classfile with ScalaSignature attribute: $clazz") + unpickler.unpickle(bytes, 0, clazz, staticModule, file.name) + } else if (!isScalaRaw && innersStart != -1) { + in.bp = innersStart + val entries = u2 + for (i <- 0 until entries) { + val innerIndex, outerIndex, nameIndex = u2 + val jflags = readInnerClassFlags() + if (innerIndex != 0 && outerIndex != 0 && nameIndex != 0) + innerClasses add InnerClassEntry(pool.getClassName(innerIndex), pool.getClassName(outerIndex), pool.getName(nameIndex), jflags) } } in.bp = oldbp } /** An entry in the InnerClasses attribute of this class file. */ - case class InnerClassEntry(external: Int, outer: Int, name: Int, jflags: JavaAccFlags) { - def externalName = pool getClassName external - def outerName = pool getClassName outer - def originalName = pool getName name + case class InnerClassEntry(external: NameOrString, outer: NameOrString, name: NameOrString, jflags: JavaAccFlags) { + def externalName = external.value + def outerName = outer.value + def originalName = name.name def isModule = originalName.isTermName def scope = if (jflags.isStatic) staticScope else instanceScope def enclosing = if (jflags.isStatic) enclModule else enclClass // The name of the outer class, without its trailing $ if it has one. - private def strippedOuter = outerName.dropModule + private def strippedOuter = outerName.stripSuffix(nme.MODULE_SUFFIX_STRING) private def isInner = innerClasses contains strippedOuter private def enclClass = if (isInner) innerClasses innerSymbol strippedOuter else classNameToSymbol(strippedOuter) private def enclModule = enclClass.companionModule @@ -1259,10 +1208,10 @@ abstract class ClassfileParser { * If the given name is not an inner class, it returns the symbol found in `definitions`. */ object innerClasses { - private val inners = mutable.HashMap[Name, InnerClassEntry]() + private val inners = mutable.HashMap[String, InnerClassEntry]() - def contains(name: Name) = inners contains name - def getEntry(name: Name) = inners get name + def contains(name: String) = inners contains name + def getEntry(name: String) = inners get name def entries = inners.values def add(entry: InnerClassEntry): Unit = { @@ -1272,7 +1221,7 @@ abstract class ClassfileParser { } inners(entry.externalName) = entry } - def innerSymbol(externalName: Name): Symbol = this getEntry externalName match { + def innerSymbol(externalName: String): Symbol = this getEntry externalName match { case Some(entry) => innerSymbol(entry) case _ => NoSymbol } @@ -1301,6 +1250,128 @@ abstract class ClassfileParser { sym setInfo createFromClonedSymbols(alias.initialize.typeParams, alias.tpe)(typeFun) } } + private class ParamNames(val names: Array[NameOrString], val access: Array[Int]) { + assert(names.length == access.length) + def length = names.length + } + private abstract class JavaTypeCompleter extends LazyType { + var constant: Constant = _ + var sig: String = _ + var paramNames: ParamNames = _ + var exceptions: List[NameOrString] = Nil + } + private final class ClassTypeCompleter(name: Name, jflags: JavaAccFlags, parent: NameOrString, ifaces: List[NameOrString]) extends JavaTypeCompleter { + override def complete(sym: symbolTable.Symbol): Unit = { + val info = if (sig != null) sigToType(sym, sig) else { + val superType = + if (parent == null) AnyClass.tpe_* + else if (jflags.isAnnotation) { u2; AnnotationClass.tpe } + else getClassSymbol(parent.value).tpe_* + var ifacesTypes = ifaces.filterNot(_ eq null).map(x => getClassSymbol(x.value).tpe_*) + if (jflags.isAnnotation) ifacesTypes ::= ClassfileAnnotationClass.tpe + ClassInfoType(superType :: ifacesTypes, instanceScope, clazz) + } + sym.setInfo(info) + } + } + + private final class MemberTypeCompleter(name: Name, jflags: JavaAccFlags, descriptor: String) extends JavaTypeCompleter { + override def isJavaVarargsMethod: Boolean = jflags.isVarargs + override def javaThrownExceptions: List[Symbol] = exceptions.map(e => classNameToSymbol(e.value)) + override def complete(sym: symbolTable.Symbol): Unit = { + def descriptorInfo = sigToType(sym, descriptor) + val hasOuterParam = (name == nme.CONSTRUCTOR) && (descriptorInfo match { + case MethodType(params, restpe) => + // if this is a non-static inner class, remove the explicit outer parameter + innerClasses getEntry currentClass match { + case Some(entry) if !entry.jflags.isStatic => + /* About `clazz.owner.hasPackageFlag` below: scala/bug#5957 + * For every nested java class A$B, there are two symbols in the scala compiler. + * 1. created by SymbolLoader, because of the existence of the A$B.class file, owner: package + * 2. created by ClassfileParser of A when reading the inner classes, owner: A + * If symbol 1 gets completed (e.g. because the compiled source mentions `A$B`, not `A#B`), the + * ClassfileParser for 1 executes, and clazz.owner is the package. + */ + assert(params.head.tpe.typeSymbol == clazz.owner || clazz.owner.hasPackageFlag, "" + params.head.tpe.typeSymbol + ": " + clazz.owner) + true + case _ => + false + } + case _ => false + }) + + val info = if (sig != null) { + sigToType(sym, sig) + } else if (name == nme.CONSTRUCTOR) { + descriptorInfo match { + case MethodType(params, restpe) => + val paramsNoOuter = if (hasOuterParam) params.tail else params + val newParams = paramsNoOuter match { + case (init :+ tail) if jflags.isSynthetic => + // scala/bug#7455 strip trailing dummy argument ("access constructor tag") from synthetic constructors which + // are added when an inner class needs to access a private constructor. + init + case _ => + paramsNoOuter + } + MethodType(newParams, clazz.tpe) + case info => info + } + } else { + descriptorInfo + } + if (constant != null) { + val c1 = convertTo(constant, info.resultType) + if (c1 ne null) sym.setInfo(ConstantType(c1)) + else { + devWarning(s"failure to convert $constant to ${info.resultType}") + sym.setInfo(info) + } + } else { + sym.setInfo(if (sym.isMethod && jflags.isVarargs) arrayToRepeated(info) else info) + } + + for (e <- exceptions) { + // we call initialize due to the fact that we call Symbol.isMonomorphicType in addThrowsAnnotation + // and that method requires Symbol to be forced to give the right answers, see scala/bug#7107 for details + val cls = getClassSymbol(e.value) + sym withAnnotation AnnotationInfo.lazily { + val throwableTpe = cls.tpe_* + AnnotationInfo(appliedType(ThrowsClass, throwableTpe), List(Literal(Constant(throwableTpe))), Nil) + } + } + + // Note: the info may be overwritten later with a generic signature + // parsed from SignatureATTR + if (paramNames != null) { + import scala.tools.asm.Opcodes.ACC_SYNTHETIC + + if (sym.hasRawInfo && sym.isMethod) { + val paramNamesNoOuter = (if (hasOuterParam) 1 else 0) to paramNames.length + val params = sym.rawInfo.params + foreach2(paramNamesNoOuter.toList, params) { + case (i, param) => + val isSynthetic = (paramNames.access(i) & ACC_SYNTHETIC) != 0 + if (!isSynthetic) { + param.name = paramNames.names(i).name.toTermName.encode + param.resetFlag(SYNTHETIC) + } + } + // there's not anything we can do, but it's slightly worrisome + devWarningIf(!sameLength(paramNamesNoOuter.toList, params)) { + sm"""MethodParameters length mismatch while parsing $sym: + | rawInfo.params: ${sym.rawInfo.params}""" + } + } + } + } + private def convertTo(c: Constant, pt: Type): Constant = { + if (pt.typeSymbol == BooleanClass && c.tag == IntTag) + Constant(c.value != 0) + else + c convertTo pt + } + } def skipAttributes() { var attrCount: Int = u2 diff --git a/src/compiler/scala/tools/nsc/symtab/classfile/DataReader.scala b/src/compiler/scala/tools/nsc/symtab/classfile/DataReader.scala new file mode 100644 index 00000000000..8c1287ac0df --- /dev/null +++ b/src/compiler/scala/tools/nsc/symtab/classfile/DataReader.scala @@ -0,0 +1,68 @@ +/* + * Scala (https://www.scala-lang.org) + * + * Copyright EPFL and Lightbend, Inc. + * + * Licensed under Apache License 2.0 + * (http://www.apache.org/licenses/LICENSE-2.0). + * + * See the NOTICE file distributed with this work for + * additional information regarding copyright ownership. + */ + +package scala.tools.nsc.symtab.classfile + +trait DataReader { + + def bp: Int + def bp_=(i: Int): Unit + + /** read a byte + */ + @throws(classOf[IndexOutOfBoundsException]) + def nextByte: Byte + + /** read some bytes + */ + def nextBytes(len: Int): Array[Byte] + + /** read a character + */ + def nextChar: Char + + /** read an integer + */ + def nextInt: Int + + /** extract a character at position bp from buf + */ + def getChar(mybp: Int): Char + + /** extract an integer at position bp from buf + */ + def getByte(mybp: Int): Byte + + def getBytes(mybp: Int, bytes: Array[Byte]): Unit + + /** extract an integer at position bp from buf + */ + def getInt(mybp: Int): Int + + /** extract a long integer at position bp from buf + */ + def getLong(mybp: Int): Long + + /** extract a float at position bp from buf + */ + def getFloat(mybp: Int): Float + + /** extract a double at position bp from buf + */ + def getDouble(mybp: Int): Double + + def getUTF(mybp: Int, len: Int): String + + /** skip next 'n' bytes + */ + def skip(n: Int): Unit +} diff --git a/src/compiler/scala/tools/nsc/symtab/classfile/ReusableDataReader.scala b/src/compiler/scala/tools/nsc/symtab/classfile/ReusableDataReader.scala new file mode 100644 index 00000000000..8bbbc4a3cce --- /dev/null +++ b/src/compiler/scala/tools/nsc/symtab/classfile/ReusableDataReader.scala @@ -0,0 +1,156 @@ +/* + * Scala (https://www.scala-lang.org) + * + * Copyright EPFL and Lightbend, Inc. + * + * Licensed under Apache License 2.0 + * (http://www.apache.org/licenses/LICENSE-2.0). + * + * See the NOTICE file distributed with this work for + * additional information regarding copyright ownership. + */ + +package scala.tools.nsc.symtab.classfile + +import java.io.{ByteArrayInputStream, DataInputStream, InputStream} +import java.nio.channels.Channels +import java.nio.{BufferUnderflowException, ByteBuffer} + +final class ReusableDataReader() extends DataReader { + private[this] var data = new Array[Byte](32768) + private[this] var bb: ByteBuffer = ByteBuffer.wrap(data) + private[this] var size = 0 + private[this] val reader: DataInputStream = { + val stream = new InputStream { + override def read(): Int = try { + bb.get & 0xff + } catch { + case _: BufferUnderflowException => -1 + } + + override def read(b: Array[Byte], off: Int, len: Int): Int = { + val pos = bb.position() + bb.get(b, off, len) + bb.position() - pos + } + + override def markSupported(): Boolean = false + } + new DataInputStream(stream) + } + + private def nextPositivePowerOfTwo(target: Int): Int = 1 << -Integer.numberOfLeadingZeros(target - 1) + + def reset(file: scala.reflect.io.AbstractFile): this.type = { + this.size = 0 + file.sizeOption match { + case Some(size) => + if (size > data.length) { + data = new Array[Byte](nextPositivePowerOfTwo(size)) + } else { + java.util.Arrays.fill(data, 0.toByte) + } + val input = file.input + try { + var endOfInput = false + while (!endOfInput) { + val remaining = data.length - this.size + if (remaining == 0) endOfInput = true + else { + val read = input.read(data, this.size, remaining) + if (read < 0) endOfInput = true + else this.size += read + } + } + bb = ByteBuffer.wrap(data, 0, size) + } finally { + input.close() + } + case None => + val input = file.input + try { + var endOfInput = false + while (!endOfInput) { + val remaining = data.length - size + if (remaining == 0) { + data = java.util.Arrays.copyOf(data, nextPositivePowerOfTwo(size)) + } + val read = input.read(data, this.size, data.length - this.size) + if (read < 0) endOfInput = true + else this.size += read + } + bb = ByteBuffer.wrap(data, 0, size) + } finally { + input.close() + } + } + this + } + + @throws(classOf[IndexOutOfBoundsException]) + def nextByte: Byte = bb.get + + def nextBytes(len: Int): Array[Byte] = { + val result = new Array[Byte](len) + reader.readFully(result) + result + } + + def nextChar: Char = bb.getChar() + + def nextInt: Int = bb.getInt() + + def getChar(mybp: Int): Char = { + bb.getChar(mybp) + } + + def getInt(mybp: Int): Int = { + bb.getInt(mybp) + } + + def getLong(mybp: Int): Long = { + bb.getLong(mybp) + } + + def getFloat(mybp: Int): Float = { + bb.getFloat(mybp) + } + + def getDouble(mybp: Int): Double = { + bb.getDouble(mybp) + } + + def skip(n: Int): Unit = { + bb.position(bb.position() + n) + } + def bp: Int = bb.position() + def bp_=(i: Int): Unit = { + try { + bb.position(i) + } catch { + case ex: IllegalArgumentException => + throw ex + } + } + + def getByte(mybp: Int): Byte = { + bb.get(mybp) + } + def getBytes(mybp: Int, bytes: Array[Byte]): Unit = { + val saved = bb.position() + bb.position(mybp) + try reader.readFully(bytes) + finally bb.position(saved) + } + def getUTF(mybp: Int, len: Int): String = { + val saved = bb.position() + val savedLimit = bb.limit() + bb.position(mybp) + bb.limit(mybp + len) + try reader.readUTF() + finally { + bb.limit(savedLimit) + bb.position(saved) + } + } +} diff --git a/src/reflect/scala/reflect/internal/Definitions.scala b/src/reflect/scala/reflect/internal/Definitions.scala index 2828db3e01d..eba017a6ae8 100644 --- a/src/reflect/scala/reflect/internal/Definitions.scala +++ b/src/reflect/scala/reflect/internal/Definitions.scala @@ -192,11 +192,11 @@ trait Definitions extends api.StandardDefinitions { // It becomes tricky to create dedicated objects for other symbols because // of initialization order issues. - lazy val JavaLangPackage = getPackage(TermName("java.lang")) + lazy val JavaLangPackage = getPackage("java.lang") lazy val JavaLangPackageClass = JavaLangPackage.moduleClass.asClass - lazy val ScalaPackage = getPackage(TermName("scala")) + lazy val ScalaPackage = getPackage("scala") lazy val ScalaPackageClass = ScalaPackage.moduleClass.asClass - lazy val RuntimePackage = getPackage(TermName("scala.runtime")) + lazy val RuntimePackage = getPackage("scala.runtime") lazy val RuntimePackageClass = RuntimePackage.moduleClass.asClass def javaTypeToValueClass(jtype: Class[_]): Symbol = jtype match { @@ -292,7 +292,7 @@ trait Definitions extends api.StandardDefinitions { // top types lazy val AnyClass = enterNewClass(ScalaPackageClass, tpnme.Any, Nil, ABSTRACT) markAllCompleted lazy val AnyRefClass = newAlias(ScalaPackageClass, tpnme.AnyRef, ObjectTpe) markAllCompleted - lazy val ObjectClass = getRequiredClass(sn.Object.toString) + lazy val ObjectClass = getRequiredClass("java.lang.Object") // Cached types for core monomorphic classes lazy val AnyRefTpe = AnyRefClass.tpe @@ -343,12 +343,12 @@ trait Definitions extends api.StandardDefinitions { // exceptions and other throwables lazy val ClassCastExceptionClass = requiredClass[ClassCastException] - lazy val IndexOutOfBoundsExceptionClass = getClassByName(sn.IOOBException) - lazy val InvocationTargetExceptionClass = getClassByName(sn.InvTargetException) + lazy val IndexOutOfBoundsExceptionClass = getClassByName("java.lang.IndexOutOfBoundsException") + lazy val InvocationTargetExceptionClass = getClassByName("java.lang.reflect.InvocationTargetException") lazy val MatchErrorClass = requiredClass[MatchError] lazy val NonLocalReturnControlClass = requiredClass[scala.runtime.NonLocalReturnControl[_]] - lazy val NullPointerExceptionClass = getClassByName(sn.NPException) - lazy val ThrowableClass = getClassByName(sn.Throwable) + lazy val NullPointerExceptionClass = getClassByName("java.lang.NullPointerException") + lazy val ThrowableClass = getClassByName("java.lang.Throwable") lazy val UninitializedErrorClass = requiredClass[UninitializedFieldError] lazy val IllegalArgExceptionClass = requiredClass[IllegalArgumentException] @@ -422,7 +422,10 @@ trait Definitions extends api.StandardDefinitions { def isByName(param: Symbol) = isByNameParamType(param.tpe_*) def isCastSymbol(sym: Symbol) = sym == Any_asInstanceOf || sym == Object_asInstanceOf - def isJavaVarArgsMethod(m: Symbol) = m.isMethod && isJavaVarArgs(m.info.params) + def isJavaVarArgsMethod(m: Symbol) = m.isMethod && (m.rawInfo match { + case completer: LazyType => completer.isJavaVarargsMethod + case _ => isJavaVarArgs(m.info.params) + }) def isJavaVarArgs(params: Seq[Symbol]) = !params.isEmpty && isJavaRepeatedParamType(params.last.tpe) def isScalaVarArgs(params: Seq[Symbol]) = !params.isEmpty && isScalaRepeatedParamType(params.last.tpe) def isVarArgsList(params: Seq[Symbol]) = !params.isEmpty && isRepeatedParamType(params.last.tpe) @@ -488,7 +491,7 @@ trait Definitions extends api.StandardDefinitions { // reflection / structural types lazy val SoftReferenceClass = requiredClass[java.lang.ref.SoftReference[_]] - lazy val MethodClass = getClassByName(sn.MethodAsObject) + lazy val MethodClass = getClassByName("java.lang.reflect.Method") lazy val EmptyMethodCacheClass = requiredClass[scala.runtime.EmptyMethodCache] lazy val MethodCacheClass = requiredClass[scala.runtime.MethodCache] def methodCache_find = getMemberMethod(MethodCacheClass, nme.find_) @@ -1219,7 +1222,7 @@ trait Definitions extends api.StandardDefinitions { // Trying to allow for deprecated locations sym.isAliasType && isMetaAnnotation(sym.info.typeSymbol) ) - lazy val metaAnnotations: Set[Symbol] = getPackage(TermName("scala.annotation.meta")).info.members filter (_ isSubClass StaticAnnotationClass) toSet + lazy val metaAnnotations: Set[Symbol] = getPackage("scala.annotation.meta").info.members filter (_ isSubClass StaticAnnotationClass) toSet // According to the scala.annotation.meta package object: // * By default, annotations on (`val`-, `var`- or plain) constructor parameters diff --git a/src/reflect/scala/reflect/internal/Mirrors.scala b/src/reflect/scala/reflect/internal/Mirrors.scala index befaa49175a..0ca0794600a 100644 --- a/src/reflect/scala/reflect/internal/Mirrors.scala +++ b/src/reflect/scala/reflect/internal/Mirrors.scala @@ -46,19 +46,23 @@ trait Mirrors extends api.Mirrors { } /** Todo: organize similar to mkStatic in scala.reflect.Base */ - private def getModuleOrClass(path: Name, len: Int): Symbol = { - val point = path lastPos('.', len - 1) + private def getModuleOrClass(path: Name, len: Int): Symbol = + getModuleOrClass(path.toString, len, path.newName(_)) + + private def getModuleOrClass(path: String, len: Int, toName: String => Name): Symbol = { + val point = path lastIndexOf ('.', len - 1) val owner = - if (point > 0) getModuleOrClass(path.toTermName, point) + if (point > 0) getModuleOrClass(path, point, newTermName(_)) else RootClass - val name = path subName (point + 1, len) + + val name = toName(path.substring(point + 1, len)) val sym = owner.info member name - val result = if (path.isTermName) sym.suchThat(_ hasFlag MODULE) else sym + val result = if (name.isTermName) sym.suchThat(_ hasFlag MODULE) else sym if (result != NoSymbol) result else { if (settings.debug) { log(sym.info); log(sym.info.members) }//debug thisMirror.missingHook(owner, name) orElse { - MissingRequirementError.notFound((if (path.isTermName) "object " else "class ")+path+" in "+thisMirror) + MissingRequirementError.notFound((if (name.isTermName) "object " else "class ")+path+" in "+thisMirror) } } } @@ -69,8 +73,8 @@ trait Mirrors extends api.Mirrors { * Unlike `getModuleOrClass`, this function * loads unqualified names from the root package. */ - private def getModuleOrClass(path: Name): Symbol = - getModuleOrClass(path, path.length) + private def getModuleOrClass(path: String, toName: String => Name): Symbol = + getModuleOrClass(path, path.length, toName) /** If you're looking for a class, pass a type name. * If a module, a term name. @@ -78,10 +82,10 @@ trait Mirrors extends api.Mirrors { * Unlike `getModuleOrClass`, this function * loads unqualified names from the empty package. */ - private def staticModuleOrClass(path: Name): Symbol = { - val isPackageless = path.pos('.') == path.length - if (isPackageless) EmptyPackageClass.info decl path - else getModuleOrClass(path) + private def staticModuleOrClass(path: String, toName: String => Name): Symbol = { + val isPackageless = !path.contains('.') + if (isPackageless) EmptyPackageClass.info decl toName(path) + else getModuleOrClass(path, toName) } protected def mirrorMissingHook(owner: Symbol, name: Name): Symbol = NoSymbol @@ -104,28 +108,41 @@ trait Mirrors extends api.Mirrors { } } + @deprecated("Use overload that accepts a String.", "2.13.0") def getClassByName(fullname: Name): ClassSymbol = - ensureClassSymbol(fullname.toString, getModuleOrClass(fullname.toTypeName)) + ensureClassSymbol(fullname.toString, getModuleOrClass(fullname.toString, fullname.length, newTypeName(_))) + + def getClassByName(fullname: String): ClassSymbol = + getRequiredClass(fullname) + + // TODO_NAMES + def getRequiredClass(fullname: String, toName: String => Name): ClassSymbol = + ensureClassSymbol(fullname, getModuleOrClass(fullname, fullname.length, toName)) def getRequiredClass(fullname: String): ClassSymbol = - getClassByName(newTypeNameCached(fullname)) + ensureClassSymbol(fullname, getModuleOrClass(fullname, fullname.length, newTypeName(_))) def requiredClass[T: ClassTag] : ClassSymbol = - getRequiredClass(erasureName[T]) + getRequiredClass(erasureName[T], newTypeName(_)) def getClassIfDefined(fullname: String): Symbol = - getClassIfDefined(newTypeNameCached(fullname)) + getClassIfDefined(fullname, newTypeName(_)) + @deprecated("Use overload that accepts a String.", "2.13.0") def getClassIfDefined(fullname: Name): Symbol = wrapMissing(getClassByName(fullname.toTypeName)) + // TODO_NAMES + def getClassIfDefined(fullname: String, toName: String => Name): Symbol = + wrapMissing(getRequiredClass(fullname, toName)) + /** @inheritdoc * * Unlike getClassByName/getRequiredClass this function can also load packageless symbols. * Compiler might ignore them, but they should be loadable with macros. */ override def staticClass(fullname: String): ClassSymbol = - try ensureClassSymbol(fullname, staticModuleOrClass(newTypeNameCached(fullname))) + try ensureClassSymbol(fullname, staticModuleOrClass(fullname, newTypeName(_))) catch { case mre: MissingRequirementError => throw new ScalaReflectionException(mre.msg) } /************************ loaders of module symbols ************************/ @@ -136,11 +153,15 @@ trait Mirrors extends api.Mirrors { case _ => MissingRequirementError.notFound("object " + fullname) } + @deprecated("Use overload that accepts a String.", "2.13.0") def getModuleByName(fullname: Name): ModuleSymbol = - ensureModuleSymbol(fullname.toString, getModuleOrClass(fullname.toTermName), allowPackages = true) + getModuleByName(fullname.toString) + + def getModuleByName(fullname: String): ModuleSymbol = + ensureModuleSymbol(fullname, getModuleOrClass(fullname, fullname.length, newTermName(_)), allowPackages = true) def getRequiredModule(fullname: String): ModuleSymbol = - getModuleByName(newTermNameCached(fullname)) + getModuleByName(fullname) // TODO: What syntax do we think should work here? Say you have an object // like scala.Predef. You can't say requiredModule[scala.Predef] since there's @@ -153,10 +174,11 @@ trait Mirrors extends api.Mirrors { getRequiredModule(erasureName[T] stripSuffix "$") def getModuleIfDefined(fullname: String): Symbol = - getModuleIfDefined(newTermNameCached(fullname)) + wrapMissing(getModuleByName(fullname)) + @deprecated("Use overload that accepts a String.", "2.13.0") def getModuleIfDefined(fullname: Name): Symbol = - wrapMissing(getModuleByName(fullname.toTermName)) + getModuleIfDefined(fullname.toString) /** @inheritdoc * @@ -164,7 +186,7 @@ trait Mirrors extends api.Mirrors { * Compiler might ignore them, but they should be loadable with macros. */ override def staticModule(fullname: String): ModuleSymbol = - try ensureModuleSymbol(fullname, staticModuleOrClass(newTermNameCached(fullname)), allowPackages = false) + try ensureModuleSymbol(fullname, staticModuleOrClass(fullname, newTermName(_)), allowPackages = false) catch { case mre: MissingRequirementError => throw new ScalaReflectionException(mre.msg) } /************************ loaders of package symbols ************************/ @@ -175,8 +197,11 @@ trait Mirrors extends api.Mirrors { case _ => MissingRequirementError.notFound("package " + fullname) } + @deprecated("Use overload that accepts a String.", "2.13.0") def getPackage(fullname: TermName): ModuleSymbol = - ensurePackageSymbol(fullname.toString, getModuleOrClass(fullname), allowModules = true) + getPackage(fullname.toString) + def getPackage(fullname: String): ModuleSymbol = + ensurePackageSymbol(fullname, getModuleOrClass(fullname, newTermName(_)), allowModules = true) def getPackageIfDefined(fullname: TermName): Symbol = wrapMissing(getPackage(fullname)) @@ -198,7 +223,7 @@ trait Mirrors extends api.Mirrors { wrapMissing(getPackageObject(fullname)) override def staticPackage(fullname: String): ModuleSymbol = - try ensurePackageSymbol(fullname.toString, getModuleOrClass(newTermNameCached(fullname)), allowModules = false) + try ensurePackageSymbol(fullname.toString, getModuleOrClass(fullname, fullname.length, newTermName(_)), allowModules = false) catch { case mre: MissingRequirementError => throw new ScalaReflectionException(mre.msg) } /************************ helpers ************************/ diff --git a/src/reflect/scala/reflect/internal/Names.scala b/src/reflect/scala/reflect/internal/Names.scala index 7e19e72e9ea..e74257dde1d 100644 --- a/src/reflect/scala/reflect/internal/Names.scala +++ b/src/reflect/scala/reflect/internal/Names.scala @@ -52,6 +52,8 @@ trait Names extends api.Names { /** Hashtable for finding type names quickly. */ private val typeHashtable = new Array[TypeName](HASH_SIZE) + final def allNames(): Iterator[TermName] = termHashtable.iterator.filter(_ ne null).flatMap(n => Iterator.iterate(n)(_.next).takeWhile(_ ne null)) + private def hashValue(cs: Array[Char], offset: Int, len: Int): Int = { var h = 0 var i = 0 diff --git a/src/reflect/scala/reflect/internal/StdNames.scala b/src/reflect/scala/reflect/internal/StdNames.scala index 6428d83cdf1..75935982a85 100644 --- a/src/reflect/scala/reflect/internal/StdNames.scala +++ b/src/reflect/scala/reflect/internal/StdNames.scala @@ -1170,21 +1170,15 @@ trait StdNames { protected val stringToTypeName = null protected implicit def createNameType(s: String): TypeName = newTypeNameCached(s) - final val BoxedBoolean: TypeName = "java.lang.Boolean" - final val BoxedByte: TypeName = "java.lang.Byte" - final val BoxedCharacter: TypeName = "java.lang.Character" - final val BoxedDouble: TypeName = "java.lang.Double" - final val BoxedFloat: TypeName = "java.lang.Float" - final val BoxedInteger: TypeName = "java.lang.Integer" - final val BoxedLong: TypeName = "java.lang.Long" - final val BoxedNumber: TypeName = "java.lang.Number" - final val BoxedShort: TypeName = "java.lang.Short" - final val IOOBException: TypeName = "java.lang.IndexOutOfBoundsException" - final val InvTargetException: TypeName = "java.lang.reflect.InvocationTargetException" - final val MethodAsObject: TypeName = "java.lang.reflect.Method" - final val NPException: TypeName = "java.lang.NullPointerException" - final val Object: TypeName = "java.lang.Object" - final val Throwable: TypeName = "java.lang.Throwable" + final val BoxedBoolean: String = "java.lang.Boolean" + final val BoxedByte: String = "java.lang.Byte" + final val BoxedCharacter: String = "java.lang.Character" + final val BoxedDouble: String = "java.lang.Double" + final val BoxedFloat: String = "java.lang.Float" + final val BoxedInteger: String = "java.lang.Integer" + final val BoxedLong: String = "java.lang.Long" + final val BoxedNumber: String = "java.lang.Number" + final val BoxedShort: String = "java.lang.Short" final val GetCause: TermName = newTermName("getCause") final val GetClass: TermName = newTermName("getClass") @@ -1197,7 +1191,7 @@ trait StdNames { final val AltMetafactory: TermName = newTermName("altMetafactory") final val Bootstrap: TermName = newTermName("bootstrap") - val Boxed = immutable.Map[TypeName, TypeName]( + val Boxed = immutable.Map[TypeName, String]( tpnme.Boolean -> BoxedBoolean, tpnme.Byte -> BoxedByte, tpnme.Char -> BoxedCharacter, diff --git a/src/reflect/scala/reflect/internal/Symbols.scala b/src/reflect/scala/reflect/internal/Symbols.scala index 7982e71000c..8d9d87c7c2a 100644 --- a/src/reflect/scala/reflect/internal/Symbols.scala +++ b/src/reflect/scala/reflect/internal/Symbols.scala @@ -24,7 +24,7 @@ import scala.collection.mutable.ListBuffer import util.{ Statistics, shortClassOfInstance, StatisticsStatics } import Flags._ import scala.annotation.tailrec -import scala.reflect.io.{ AbstractFile, NoAbstractFile } +import scala.reflect.io.{AbstractFile, NoAbstractFile} import Variance._ trait Symbols extends api.Symbols { self: SymbolTable => @@ -3030,7 +3030,14 @@ trait Symbols extends api.Symbols { self: SymbolTable => loop(info) } - override def exceptions = for (ThrownException(tp) <- annotations) yield tp.typeSymbol + override def exceptions = { + rawInfo match { + case lt: LazyType if isJava => + lt.javaThrownExceptions + case _ => + for (ThrownException(tp) <- annotations) yield tp.typeSymbol + } + } } implicit val MethodSymbolTag = ClassTag[MethodSymbol](classOf[MethodSymbol]) diff --git a/src/reflect/scala/reflect/internal/Types.scala b/src/reflect/scala/reflect/internal/Types.scala index 92ac84ff876..6710f0abbe7 100644 --- a/src/reflect/scala/reflect/internal/Types.scala +++ b/src/reflect/scala/reflect/internal/Types.scala @@ -1540,53 +1540,9 @@ trait Types throw new TypeError("illegal cyclic inheritance involving " + tpe.typeSymbol) } - object baseClassesCycleMonitor { - private var open: List[Symbol] = Nil - @inline private def cycleLog(msg: => String) { - if (settings.debug) - Console.err.println(msg) - } - def size = open.size - def push(clazz: Symbol) { - cycleLog("+ " + (" " * size) + clazz.fullNameString) - open ::= clazz - } - def pop(clazz: Symbol) { - assert(open.head eq clazz, (clazz, open)) - open = open.tail - } - def isOpen(clazz: Symbol) = open contains clazz - } - protected def defineBaseClassesOfCompoundType(tpe: CompoundType) { - def define() = defineBaseClassesOfCompoundType(tpe, force = false) - if (!breakCycles || isPastTyper) define() - else tpe match { - // non-empty parents helpfully excludes all package classes - case tpe @ ClassInfoType(_ :: _, _, clazz) if !clazz.isAnonOrRefinementClass => - // Cycle: force update - if (baseClassesCycleMonitor isOpen clazz) - defineBaseClassesOfCompoundType(tpe, force = true) - else { - baseClassesCycleMonitor push clazz - try define() - finally baseClassesCycleMonitor pop clazz - } - case _ => - define() - } - } - private def defineBaseClassesOfCompoundType(tpe: CompoundType, force: Boolean) { val period = tpe.baseClassesPeriod - if (period == currentPeriod) { - if (force && breakCycles) { - def what = tpe.typeSymbol + " in " + tpe.typeSymbol.owner.fullNameString - val bcs = computeBaseClasses(tpe) - tpe.baseClassesCache = bcs - warning(s"Breaking cycle in base class computation of $what ($bcs)") - } - } - else { + if (period != currentPeriod) { tpe.baseClassesPeriod = currentPeriod if (!isValidForBaseClasses(period)) { val start = if (StatisticsStatics.areSomeColdStatsEnabled) statistics.pushTimer(typeOpsStack, baseClassesNanos) else null @@ -3555,6 +3511,8 @@ trait Types override def complete(sym: Symbol) override def safeToString = "" override def kind = "LazyType" + def isJavaVarargsMethod: Boolean = false + def javaThrownExceptions: List[Symbol] = Nil } /** A marker trait representing an as-yet unevaluated type diff --git a/src/reflect/scala/reflect/io/AbstractFile.scala b/src/reflect/scala/reflect/io/AbstractFile.scala index 714f4f4b527..996725a65a9 100644 --- a/src/reflect/scala/reflect/io/AbstractFile.scala +++ b/src/reflect/scala/reflect/io/AbstractFile.scala @@ -17,6 +17,7 @@ package io import java.io.{ IOException, InputStream, OutputStream, BufferedOutputStream, ByteArrayOutputStream } import java.io.{ File => JFile } import java.net.URL +import java.nio.ByteBuffer /** * An abstraction over files for use in the reflection/compiler libraries. diff --git a/src/reflect/scala/reflect/io/PlainFile.scala b/src/reflect/scala/reflect/io/PlainFile.scala index 75ba6e85202..cb1f73b4164 100644 --- a/src/reflect/scala/reflect/io/PlainFile.scala +++ b/src/reflect/scala/reflect/io/PlainFile.scala @@ -14,6 +14,10 @@ package scala package reflect package io +import java.nio.ByteBuffer +import java.nio.file.StandardOpenOption +import java.util + /** ''Note: This library is considered experimental and should not be used unless you know what you are doing.'' */ class PlainDirectory(givenPath: Directory) extends PlainFile(givenPath) { override def isDirectory = true diff --git a/src/reflect/scala/reflect/runtime/JavaUniverseForce.scala b/src/reflect/scala/reflect/runtime/JavaUniverseForce.scala index 0b4d7131fbe..264a3cd9afd 100644 --- a/src/reflect/scala/reflect/runtime/JavaUniverseForce.scala +++ b/src/reflect/scala/reflect/runtime/JavaUniverseForce.scala @@ -162,7 +162,6 @@ trait JavaUniverseForce { self: runtime.JavaUniverse => this.SuperType this.TypeBounds this.CompoundType - this.baseClassesCycleMonitor this.RefinedType this.ClassInfoType this.ConstantType diff --git a/test/files/jvm/throws-annot-from-java.check b/test/files/jvm/throws-annot-from-java.check index bf639260e77..4a4bd6ad211 100644 --- a/test/files/jvm/throws-annot-from-java.check +++ b/test/files/jvm/throws-annot-from-java.check @@ -8,10 +8,10 @@ scala> :paste // Entering paste mode (ctrl-D to finish) { - val clazz = rootMirror.getClassByName(newTermName("test.ThrowsDeclaration_2")); + val clazz = rootMirror.getClassByName("test.ThrowsDeclaration_2"); { val method = clazz.info.member(newTermName("foo")) - val throwsAnn = method.annotations.head + val throwsAnn = method.initialize.annotations.head val atp = throwsAnn.atp println("foo") println("atp.typeParams.isEmpty: " + atp.typeParams.isEmpty) @@ -21,7 +21,7 @@ scala> :paste { val method = clazz.info.member(newTermName("bar")) - val throwsAnn = method.annotations.head + val throwsAnn = method.initialize.annotations.head val Literal(const) = throwsAnn.args.head val tp = const.typeValue println("bar") @@ -37,7 +37,7 @@ atp.typeParams.isEmpty: true throws[IllegalStateException](classOf[java.lang.IllegalStateException]) bar -tp.typeParams.isEmpty: true -throws[test.PolymorphicException[_]](classOf[test.PolymorphicException]) +tp.typeParams.isEmpty: false +throws[test.PolymorphicException](classOf[test.PolymorphicException]) scala> :quit diff --git a/test/files/jvm/throws-annot-from-java/Test_3.scala b/test/files/jvm/throws-annot-from-java/Test_3.scala index de1d9845732..df62e032262 100644 --- a/test/files/jvm/throws-annot-from-java/Test_3.scala +++ b/test/files/jvm/throws-annot-from-java/Test_3.scala @@ -4,10 +4,10 @@ object Test extends ReplTest { def code = """:power :paste { - val clazz = rootMirror.getClassByName(newTermName("test.ThrowsDeclaration_2")); + val clazz = rootMirror.getClassByName("test.ThrowsDeclaration_2"); { val method = clazz.info.member(newTermName("foo")) - val throwsAnn = method.annotations.head + val throwsAnn = method.initialize.annotations.head val atp = throwsAnn.atp println("foo") println("atp.typeParams.isEmpty: " + atp.typeParams.isEmpty) @@ -17,7 +17,7 @@ object Test extends ReplTest { { val method = clazz.info.member(newTermName("bar")) - val throwsAnn = method.annotations.head + val throwsAnn = method.initialize.annotations.head val Literal(const) = throwsAnn.args.head val tp = const.typeValue println("bar") diff --git a/test/files/neg/moduleClassReference.check b/test/files/neg/moduleClassReference.check new file mode 100644 index 00000000000..1f16aeb2509 --- /dev/null +++ b/test/files/neg/moduleClassReference.check @@ -0,0 +1,4 @@ +moduleClassReference.scala:2: error: not found: value Predef$ + def foo = Predef$.MODULE$ == Predef + ^ +one error found diff --git a/test/files/neg/moduleClassReference.scala b/test/files/neg/moduleClassReference.scala new file mode 100644 index 00000000000..dbf688840e2 --- /dev/null +++ b/test/files/neg/moduleClassReference.scala @@ -0,0 +1,3 @@ +object Test { + def foo = Predef$.MODULE$ == Predef +} diff --git a/test/files/neg/t7251.check b/test/files/neg/t7251.check index 33fdafc2ee1..a17e710d367 100644 --- a/test/files/neg/t7251.check +++ b/test/files/neg/t7251.check @@ -1,4 +1,4 @@ -B_2.scala:5: error: class s.Outer$Triple$ is not a value +B_2.scala:5: error: object Outer$Triple$ is not a member of package s println( s.Outer$Triple$ ) ^ one error found diff --git a/test/files/run/compiler-asSeenFrom.scala b/test/files/run/compiler-asSeenFrom.scala index a60c2e89252..94090766582 100644 --- a/test/files/run/compiler-asSeenFrom.scala +++ b/test/files/run/compiler-asSeenFrom.scala @@ -42,7 +42,7 @@ abstract class CompilerTest extends DirectTest { } class SymsInPackage(pkgName: String) { - def pkg = rootMirror.getPackage(TermName(pkgName)) + def pkg = rootMirror.getPackage(pkgName) def classes = allMembers(pkg) filter (_.isClass) def modules = allMembers(pkg) filter (_.isModule) def symbols = classes ++ terms filterNot (_ eq NoSymbol) diff --git a/test/files/run/existentials-in-compiler.scala b/test/files/run/existentials-in-compiler.scala index e35b7231c2d..2984d81e600 100644 --- a/test/files/run/existentials-in-compiler.scala +++ b/test/files/run/existentials-in-compiler.scala @@ -74,8 +74,8 @@ package extest { } """ - override def check(source: String, unit: global.CompilationUnit) { - getPackage(TermName("extest")).moduleClass.info.decls.toList.filter(_.isType).map(_.initialize).sortBy(_.name.toString) foreach { clazz => + override def check(source: String, unit: global.CompilationUnit): Unit = { + getPackage("extest").moduleClass.info.decls.toList.filter(_.isType).map(_.initialize).sortBy(_.name.toString) foreach { clazz => exitingTyper { clazz.info println(clazz.defString) diff --git a/test/files/run/t7008-scala-defined/Impls_Macros_2.scala b/test/files/run/t7008-scala-defined/Impls_Macros_2.scala index 330db8da753..d49cfff1aa1 100644 --- a/test/files/run/t7008-scala-defined/Impls_Macros_2.scala +++ b/test/files/run/t7008-scala-defined/Impls_Macros_2.scala @@ -5,6 +5,8 @@ object Macros { def impl(c: Context) = { import c.universe._ val decls = c.typeOf[ScalaClassWithCheckedExceptions_1[_]].decls.toList + decls.foreach(_.info) + decls.foreach(_.annotations.foreach(_.tpe)) val s = decls.sortBy(_.name.toString).map(decl => (s"${decl.name}: ${decl.annotations}")).mkString(scala.compat.Platform.EOL) reify(println(c.Expr[String](Literal(Constant(s))).splice)) } diff --git a/test/files/run/t7008/Impls_Macros_2.scala b/test/files/run/t7008/Impls_Macros_2.scala index 3c6fe116ce2..e55cbbfdbf8 100644 --- a/test/files/run/t7008/Impls_Macros_2.scala +++ b/test/files/run/t7008/Impls_Macros_2.scala @@ -5,6 +5,8 @@ object Macros { def impl(c: Context) = { import c.universe._ val decls = c.typeOf[JavaClassWithCheckedExceptions_1[_]].decls.toList + decls.foreach(_.info) + decls.foreach(_.annotations.foreach(_.tpe)) val s = decls.sortBy(_.name.toString).map(decl => (s"${decl.name}: ${decl.annotations}")).mkString(scala.compat.Platform.EOL) reify(println(c.Expr[String](Literal(Constant(s))).splice)) } diff --git a/test/files/run/t7096.scala b/test/files/run/t7096.scala index f723d70abef..44485e5da1e 100644 --- a/test/files/run/t7096.scala +++ b/test/files/run/t7096.scala @@ -41,7 +41,7 @@ abstract class CompilerTest extends DirectTest { } class SymsInPackage(pkgName: String) { - def pkg = rootMirror.getPackage(TermName(pkgName)) + def pkg = rootMirror.getPackage(pkgName) def classes = allMembers(pkg) filter (_.isClass) def modules = allMembers(pkg) filter (_.isModule) def symbols = classes ++ terms filterNot (_ eq NoSymbol) diff --git a/test/files/run/t7455/Test.scala b/test/files/run/t7455/Test.scala index 2cda9225f4f..afe3f09fb57 100644 --- a/test/files/run/t7455/Test.scala +++ b/test/files/run/t7455/Test.scala @@ -23,8 +23,8 @@ object Test extends DirectTest { clazz = compiler.rootMirror.staticClass(name) constr <- clazz.info.member(termNames.CONSTRUCTOR).alternatives } { - println(constr.defString) fullyInitializeSymbol(constr) + println(constr.defString) } } } From 3b3b9dbac4f6be123cc635b48d3651164d0dcfd0 Mon Sep 17 00:00:00 2001 From: Mike Skells Date: Fri, 17 May 2019 00:33:08 +0100 Subject: [PATCH 2/7] first code of interners to bake off different implementations --- .../AbstractConcurrentNodeInterner.scala | 65 ++++ .../internal/names/AbstractNodeInterner.scala | 31 ++ .../names/ConcurrentMapNameTable.scala | 12 + .../reflect/internal/names/MapNameTable.scala | 10 + .../reflect/internal/names/NameBase.scala | 3 + .../reflect/internal/names/NameTable.scala | 16 + .../scala/reflect/internal/names/Node.scala | 292 +++++++++++++++ .../names/StrongConcurrentNodeInterner.scala | 120 ++++++ .../names/WeakConcurrentMapNameTable.scala | 63 ++++ .../WeakFixedSizeConcurrentNodeInterner.scala | 351 ++++++++++++++++++ .../internal/names/WeakMapNameTable.scala | 49 +++ .../scala/reflect/ExistingNameTable.scala | 17 + .../scala/scala/reflect/NamesBenchmark.scala | 114 ++++++ .../internal/names/ConcurrentNamesTest.scala | 33 ++ .../internal/names/ExtendedNameTest.scala | 127 +++++++ .../internal/names/NamesImplTest.scala | 67 ++++ .../internal/names/WeakNamesTest.scala | 88 +++++ 17 files changed, 1458 insertions(+) create mode 100644 src/reflect/scala/reflect/internal/names/AbstractConcurrentNodeInterner.scala create mode 100644 src/reflect/scala/reflect/internal/names/AbstractNodeInterner.scala create mode 100644 src/reflect/scala/reflect/internal/names/ConcurrentMapNameTable.scala create mode 100644 src/reflect/scala/reflect/internal/names/MapNameTable.scala create mode 100644 src/reflect/scala/reflect/internal/names/NameBase.scala create mode 100644 src/reflect/scala/reflect/internal/names/NameTable.scala create mode 100644 src/reflect/scala/reflect/internal/names/Node.scala create mode 100644 src/reflect/scala/reflect/internal/names/StrongConcurrentNodeInterner.scala create mode 100644 src/reflect/scala/reflect/internal/names/WeakConcurrentMapNameTable.scala create mode 100644 src/reflect/scala/reflect/internal/names/WeakFixedSizeConcurrentNodeInterner.scala create mode 100644 src/reflect/scala/reflect/internal/names/WeakMapNameTable.scala create mode 100644 test/benchmarks/src/main/scala/scala/reflect/ExistingNameTable.scala create mode 100644 test/benchmarks/src/main/scala/scala/reflect/NamesBenchmark.scala create mode 100644 test/junit/scala/reflect/internal/names/ConcurrentNamesTest.scala create mode 100644 test/junit/scala/reflect/internal/names/ExtendedNameTest.scala create mode 100644 test/junit/scala/reflect/internal/names/NamesImplTest.scala create mode 100644 test/junit/scala/reflect/internal/names/WeakNamesTest.scala diff --git a/src/reflect/scala/reflect/internal/names/AbstractConcurrentNodeInterner.scala b/src/reflect/scala/reflect/internal/names/AbstractConcurrentNodeInterner.scala new file mode 100644 index 00000000000..30004c11b11 --- /dev/null +++ b/src/reflect/scala/reflect/internal/names/AbstractConcurrentNodeInterner.scala @@ -0,0 +1,65 @@ +package scala.reflect.internal.names + +import java.util.concurrent.atomic.{AtomicInteger, AtomicReferenceArray, AtomicReferenceFieldUpdater} + +//object AbstractConcurrentNodeInterner { +// private val dataAccess = AtomicReferenceFieldUpdater.newUpdater(classOf[AbstractConcurrentNodeInterner[_]], classOf[AtomicReferenceArray[_]], "data") +//} + +abstract class AbstractConcurrentNodeInterner[T >: Null <: NameBase](createName: String => T) extends AbstractNodeInterner[T](createName) { + + //also serves as a synchronization point for a change to data + protected[this] val size_ = new AtomicInteger + def size = size_.get + + /** + * all values in data are live. All values entered must be only entered after all checks to intern are completed + * for child classes that resize a flag value is generally CASed into this array before a row is resized + */ + @volatile private[this] var data = new AtomicReferenceArray[N](1 << 16) + + protected final def setDataNull(): Unit = { + assert (Thread.holdsLock(size_)) + data = null + } + protected final def setData(newData: AtomicReferenceArray[N]): Unit = { + assert(newData ne null) + assert(data eq null) + assert (Thread.holdsLock(size_)) + data = newData + } + protected final def isCurrentData(data: AtomicReferenceArray[N]): Boolean = { + assert(data ne null) + assert (Thread.holdsLock(size_)) + data eq this.data + } + + + def getExistingImpl(key: String): T = { + val data = initial() + val hash = key.hashCode() + val improved = improveHash(hash) + val head: N = data.get(improved & (data.length() - 1)) + if (head eq null) null + else find(head,key, hash) + } + + protected def find(start: N, key: String, hash:Int):T + /** + * get the root of data + * + * @return + */ + protected def initial(): AtomicReferenceArray[N] = { + //volatile read + var result = data + //null indicates it is in the process of being rehashed + //updates are applied with synchronisation lock on data + if (result eq null) size.synchronized { + //when we have the lock we can guarantee that the other threads rehash is complete + result = data + assert(result ne null) + } + result + } +} \ No newline at end of file diff --git a/src/reflect/scala/reflect/internal/names/AbstractNodeInterner.scala b/src/reflect/scala/reflect/internal/names/AbstractNodeInterner.scala new file mode 100644 index 00000000000..b08d2c8d1f4 --- /dev/null +++ b/src/reflect/scala/reflect/internal/names/AbstractNodeInterner.scala @@ -0,0 +1,31 @@ +package scala.reflect.internal.names + +abstract class AbstractNodeInterner[T >: Null <: NameBase](val createName: String => T) extends NameTable[T] { + type N <: Node[T] + + def size: Int + + @inline protected final def improveHash(hash: Int) = hash ^ hash >>> 16 + protected final def hashLikeString(chars: Array[Char], offset: Int, length: Int): Int = { + var h = 0 + var index = offset + val max = offset + length + while (index < max) { + h = 31 * h + chars(index) + index += 1 + } + h + } + + + final def contains(key: String): Boolean = { + getExistingImpl(key) ne null + } + + final def get(key: String): Option[T] = { + Option(getExistingImpl(key)) + } + + def getExistingImpl(key: String): T + +} diff --git a/src/reflect/scala/reflect/internal/names/ConcurrentMapNameTable.scala b/src/reflect/scala/reflect/internal/names/ConcurrentMapNameTable.scala new file mode 100644 index 00000000000..6aa3d2ae5c3 --- /dev/null +++ b/src/reflect/scala/reflect/internal/names/ConcurrentMapNameTable.scala @@ -0,0 +1,12 @@ +package scala.reflect.internal.names + +import java.util.concurrent.ConcurrentHashMap + +class ConcurrentMapNameTable[T <: AnyRef](builder: (String) => T) extends NameTable[T] { + + override def size: Int = data.size + + private val data = new ConcurrentHashMap[String, T] + + def find(source: String): T = data.computeIfAbsent(source, (s) => builder(s)) +} diff --git a/src/reflect/scala/reflect/internal/names/MapNameTable.scala b/src/reflect/scala/reflect/internal/names/MapNameTable.scala new file mode 100644 index 00000000000..086878d5de8 --- /dev/null +++ b/src/reflect/scala/reflect/internal/names/MapNameTable.scala @@ -0,0 +1,10 @@ +package scala.reflect.internal.names + +import scala.collection.mutable + +class MapNameTable[T <: AnyRef](builder: (String) => T) extends NameTable[T] { + override def size: Int = data.size + private val data = new mutable.AnyRefMap[String, T] + + def find(source: String): T = data.getOrElseUpdate(source, builder(source)) +} diff --git a/src/reflect/scala/reflect/internal/names/NameBase.scala b/src/reflect/scala/reflect/internal/names/NameBase.scala new file mode 100644 index 00000000000..cd320f9c06b --- /dev/null +++ b/src/reflect/scala/reflect/internal/names/NameBase.scala @@ -0,0 +1,3 @@ +package scala.reflect.internal.names + +class NameBase(val id: String) extends AnyRef diff --git a/src/reflect/scala/reflect/internal/names/NameTable.scala b/src/reflect/scala/reflect/internal/names/NameTable.scala new file mode 100644 index 00000000000..eb07427befe --- /dev/null +++ b/src/reflect/scala/reflect/internal/names/NameTable.scala @@ -0,0 +1,16 @@ +package scala.reflect.internal.names + +import java.nio.charset.Charset + +abstract class NameTable[T <: AnyRef] { + def size: Int + def find(source: String): T + + //there is some room to optimise here + def find(chars: Array[Char], start: Int, count: Int): T = + find(new String(chars, start, count)) +} + +object NameTable { + val charSet = Charset.forName("UTF-8") +} \ No newline at end of file diff --git a/src/reflect/scala/reflect/internal/names/Node.scala b/src/reflect/scala/reflect/internal/names/Node.scala new file mode 100644 index 00000000000..468ae9c21b2 --- /dev/null +++ b/src/reflect/scala/reflect/internal/names/Node.scala @@ -0,0 +1,292 @@ +package scala.reflect.internal.names + +import java.lang.ref.WeakReference +import java.util.concurrent.atomic.{AtomicInteger, AtomicReferenceFieldUpdater} + +import scala.annotation.tailrec + +abstract class Node[N >: Null <: NameBase] extends AnyRef { + def name: N + + def namesString: String + + def hash: Int +} + +object StrongNode { + final def find[N >: Null <: NameBase](start: StrongNode[N], key: String, hash: Int, end: StrongNode[N]): N = { + var current = start + while ((current ne null) && (start ne end)) { + val nameString = current.namesString + if (nameString.hashCode == hash && nameString == key) return current.name + current = current.next + } + null + } + +} +final class StrongNode[N >: Null <: NameBase](val name: N, var next: StrongNode[N]) extends Node[N] { + @inline def hash: Int = namesString.hashCode() + + @inline def namesString: String = name.id + + +} + +object WeakNode { + + @inline final def find[N >: Null <: NameBase](start: WeakNode[N], key: String, hash: Int, decrementOnRemove: AtomicInteger): N = { + var current = start + var currentName = current.name + if ((currentName ne null) && currentName.id.hashCode() == hash && currentName.id == key) + currentName + else { + var prev: WeakNode[N] = current + var found: N = null + current = current.next + while ((found eq null) && (current ne null)) { + currentName = current.name + if (currentName eq null) { + val next = current.next + prev.next = next + current = next + decrementOnRemove.decrementAndGet() + } else if (currentName.id.hashCode() == hash && currentName.id == key) + found = currentName + else { + prev = current + current = current.next + } + } + found + } + } +} +final class WeakNode[N >: Null <: NameBase](k: N, @volatile var next: WeakNode[N]) extends Node[N] { + private val ref = new WeakReference(k) + + def namesString: String = { + val n = name + if (n eq null) null else n.id + } + + def name = ref.get() + def isDefined = ref.get() ne null + + def hash: Int = { + val n = name + if (n eq null) 0 else n.hashCode + } + +} + +object WeakConcurrentNode { + private val nextAccess: AtomicReferenceFieldUpdater[WeakConcurrentNode[_], WeakConcurrentNode[_]] = new WeakConcurrentNode[NameBase](null, null).nextAccess + private def nextAccessN[N>: Null <:NameBase ]: AtomicReferenceFieldUpdater[WeakConcurrentNode[N], WeakConcurrentNode[N]] = nextAccess.asInstanceOf[AtomicReferenceFieldUpdater[WeakConcurrentNode[N], WeakConcurrentNode[N]]] + + private def casNext[N>: Null <:NameBase ](prev: WeakConcurrentNode[N], expectedNext: WeakConcurrentNode[N], updateNext: WeakConcurrentNode[N]): Boolean = { + nextAccess.compareAndSet(prev, expectedNext, updateNext) + } + private val builderCache = new ThreadLocal[java.lang.StringBuilder] { + override def initialValue()= new java.lang.StringBuilder + } + + final def findNoTrimC1[N >: Null <: NameBase](start: WeakConcurrentNode[N], chars: Array[Char], charOffset: Int, charCount: Int, hash: Int, end: Node[N]): N = { + def equalsAsString(nameId:String): Boolean = { + nameId.hashCode() == hash && nameId.length == charCount && { + var offset = 0 + var equal = true + while (offset < charCount && equal) { + equal = chars(charOffset+offset) == nameId.charAt(offset) + offset +=1 + } + equal + } + } + if (start eq end) + null + else { + var current = start + var prev: WeakConcurrentNode[N] = null + + var currentName: N = null + var found: N = null + + do { + currentName = current.name + if ((currentName ne null) && equalsAsString(currentName.id)) + found = currentName + else { + prev = current + current = current.next + } + } while ((found eq null) && (current ne null) && (current ne end)) + found + } + + } + final def findNoTrimC2[N >: Null <: NameBase](start: WeakConcurrentNode[N], chars: Array[Char], charOffset: Int, charCount: Int, hash: Int, end: Node[N]): N = { + var seq:java.lang.StringBuilder = null + if (start eq end) + null + else { + var current = start + var prev: WeakConcurrentNode[N] = null + + var currentName: N = null + var found: N = null + + do { + currentName = current.name + if ((currentName ne null) && currentName.id.hashCode() == hash && currentName.id.length == charCount && { + if (seq == null) { + seq = new java.lang.StringBuilder() + seq.append(chars, charOffset, charCount) + } + currentName.id.contentEquals(seq) + }) + found = currentName + else { + prev = current + current = current.next + } + } while ((found eq null) && (current ne null) && (current ne end)) + found + } + + } + + final def findNoTrimC3[N >: Null <: NameBase](start: WeakConcurrentNode[N], chars: Array[Char], charOffset: Int, charCount: Int, hash: Int, end: Node[N]): N = { + var seq:java.lang.StringBuilder = null + if (start eq end) + null + else { + var current = start + var prev: WeakConcurrentNode[N] = null + + var currentName: N = null + var found: N = null + + do { + currentName = current.name + if ((currentName ne null) && currentName.id.hashCode() == hash && currentName.id.length == charCount && { + if (seq == null) { + seq = builderCache.get() + seq.append(chars, charOffset, charCount) + } + currentName.id.contentEquals(seq) + }) + found = currentName + else { + prev = current + current = current.next + } + } while ((found eq null) && (current ne null) && (current ne end)) + if (seq ne null) + seq.setLength(0) + found + } + + } + final def findNoTrim[N >: Null <: NameBase](start: WeakConcurrentNode[N], key: String, hash: Int, end: Node[N]): N = { + if (start eq end) + null + else { + var current = start + var prev: WeakConcurrentNode[N] = null + + var currentName: N = null + var found: N = null + + do { + currentName = current.name + if ((currentName ne null) && currentName.id.hashCode() == hash && currentName.id == key) + found = currentName + else { + prev = current + current = current.next + } + } while ((found eq null) && (current ne null) && (current ne end)) + found + } + } + + /** trim the collected nodes + * this is safe WRT other threads adding nodes ( as this occurs at the head + * It is not threadsafe WRT other thread trimming + * + * @return the count of collected nodes + */ + def trimCollected[N >: Null <: NameBase](start: WeakConcurrentNode[N]) : Int= { + var trimmed = 0 + var prev: WeakConcurrentNode[N] = start + while (prev ne null) { + val next = prev.next + val nextValue = if (next eq null) null else next.name + if ((next ne null) && (nextValue eq null)) { + //Note - using a cas here doesnt make if safe WRT other trimmers as they may overlap in the list + prev.next = next.next + trimmed += 1 + } else { + prev = next + } + } + trimmed + } + +// final def findAutoTrim[N >: Null <: Name](start: WeakConcurrentNode[N], key: String, hash: Int, end: Node[N], decrementOnRemove: AtomicInteger): N = { +// if (start eq end) +// null +// else { +// //prev refers to the previos entry with a name that has not been GCed +// var prevRef: WeakConcurrentNode[N] = null +// var prevRefName: N = null +// +// var current = start +// var currentName: N = null +// var found: N = null +// +// var skipped = 0 +// +// do { +// currentName = current.name +// while ((current ne null) && (currentName eq null)) { +// val next = current.next +// skipped += 1 +// } +// } +// //we dont care if we lost the race - someone else fixed it +// if ((prev ne null) && casNext(prev, current, next)) +// decrementOnRemove.decrementAndGet() +// //leave prev where it was +// current = next +// } else if (currentName.id.hashCode() == hash && currentName.id == key) +// found = currentName +// else { +// prev = current +// current = current.next +// } +// } while ((found eq null) && (current ne null) && (current ne end)) +// found +// } +// } +} +final class WeakConcurrentNode[N >: Null <: NameBase](private val ref: WeakReference[N], @volatile var next: WeakConcurrentNode[N]) extends Node[N] { + private def nextAccess = AtomicReferenceFieldUpdater.newUpdater(classOf[WeakConcurrentNode[_]], classOf[WeakConcurrentNode[_]], "next") + + def getAndClearNext(): WeakConcurrentNode[N] = WeakConcurrentNode.nextAccessN[N].getAndSet(this, null) + + def namesString: String = { + val n = name + if (n eq null) null else n.id + } + + def name = ref.get() + def isDefined = ref.get() ne null + + def hash: Int = { + val n = name + if (n eq null) 0 else n.hashCode + } + +} diff --git a/src/reflect/scala/reflect/internal/names/StrongConcurrentNodeInterner.scala b/src/reflect/scala/reflect/internal/names/StrongConcurrentNodeInterner.scala new file mode 100644 index 00000000000..e070a28edc0 --- /dev/null +++ b/src/reflect/scala/reflect/internal/names/StrongConcurrentNodeInterner.scala @@ -0,0 +1,120 @@ +package scala.reflect.internal.names + +import java.util.concurrent.atomic.AtomicReferenceArray + +import scala.annotation.tailrec + +object StrongConcurrentNodeInterner { + private val Flag: StrongNode[_] = new StrongNode(null, null) + + @inline private def FlagN[N >: Null <: NameBase]: StrongNode[N] = Flag.asInstanceOf[StrongNode[N]] +} + +/** + * a Node Interner that allows for concurrent access + * references to Names are held strongly + * + * Data table can only even grow + * + * Data table head is only appended with CAS + * A data table row may be modified by CAS to Flag, as no other thread can add to head when it eq Flag + * + * @param createName + * @tparam T + */ +class StrongConcurrentNodeInterner[T >: Null <: NameBase](createName: String => T) extends AbstractConcurrentNodeInterner[T](createName) { + type N = StrongNode[T] + + import StrongConcurrentNodeInterner._ + + final def find(key: String): T = { + val hash = key.hashCode + val improved = improveHash(hash) + var oldTail: N = null + + var name: T = null + do { + val data = initial() + val bucket = improved & (data.length - 1) + val head = data.get(bucket) + if (head ne Flag) { + if (head ne null) + name = StrongNode.find(head, key, hash, oldTail) + if (name eq null) { + // minor optimisation - we can skip this tail if we have to retry + // if we came to the end of the chain of nodes we dont need to search the same tail if we fail and try again + oldTail = head + name = createName(key) + val newNode = new N(name, head) + if (data.compareAndSet(bucket, head, newNode)) { + afterInsert(data) + } else name = null + } + } + } while (name eq null) + name + } + + override protected def find(head: StrongNode[T], key: String, hash: Int): T = { + StrongNode.find(head, key, hash, null) + } + + /** + * rehash and grow + */ + private def afterInsert(data: AtomicReferenceArray[N]): Unit = { + val newSize = size_.incrementAndGet() + val origLength = data.length + if (origLength < newSize && origLength < (1 << 31)) { + size_.synchronized { + if (isCurrentData(data)) grow(data) + } + } + } + + @tailrec private def grow(data: AtomicReferenceArray[N]): Unit = { + val length = data.length + + if (length < size && length < (1 << 31)) { + + //we allocate this before nulling data as the rest is non heap allocating, and it eliminates OOM as a cause of failure + val newData = new AtomicReferenceArray[N](length << 1) + val mask = length + + var head0: N = null + var head1: N = null + var sourceIdx = 0 + // if the value has changed already then its not our problem + setDataNull() + + while (sourceIdx < length) { + head0 = null + head1 = null + var tail0: N = null + var tail1: N = null + var sourceNode = data.getAndSet(sourceIdx, FlagN) + while (sourceNode ne null) { + val hash = sourceNode.hash + val improved = improveHash(hash) + if ((improved & mask) == 0) { + if (head0 eq null) head0 = sourceNode + else tail0.next = sourceNode + tail0 = sourceNode + } else { + if (head1 eq null) head1 = sourceNode + else tail1.next = sourceNode + tail1 = sourceNode + } + sourceNode = sourceNode.next + } + if (tail0 ne null) tail0.next = null + if (tail1 ne null) tail1.next = null + newData.set(sourceIdx, head0) + newData.set(sourceIdx + mask, head1) + sourceIdx += 1 + } + setData(newData) + grow(newData) + } + } +} diff --git a/src/reflect/scala/reflect/internal/names/WeakConcurrentMapNameTable.scala b/src/reflect/scala/reflect/internal/names/WeakConcurrentMapNameTable.scala new file mode 100644 index 00000000000..f6e8369054a --- /dev/null +++ b/src/reflect/scala/reflect/internal/names/WeakConcurrentMapNameTable.scala @@ -0,0 +1,63 @@ +package scala.reflect.internal.names + +import java.lang.ref.{ReferenceQueue, WeakReference} +import java.util.concurrent.ConcurrentHashMap + +import scala.annotation.tailrec + +final class WeakConcurrentMapNameTable[T >: Null <: AnyRef](builder: (String) => T) extends NameTable[T] { + + class NameRef(val key: String, name: T) extends WeakReference(name, queue) + + private def newNameRef(s: String) = new NameRef(s, builder(s)) + + private val data = new ConcurrentHashMap[String, NameRef] + private val queue = new ReferenceQueue[T] + + @tailrec def processQueue(): Unit = { + val next = queue.poll().asInstanceOf[NameRef] + if (next ne null) { + data.get(next.key) match { + case null => + case ref => + if (ref.get() eq null) + data.compute(ref.key, (_: String, existing: NameRef) => + if (existing.get eq null) null + else existing) + } + processQueue() + } + } + + override def size: Int = { + processQueue() + data.size + } + + def find(source: String): T = { + processQueue() + var res: T = null + //we assume that we usually find a match, so no lock on the fast path + var entry = data.get(source) + do { + if (entry eq null) { + val maybe = builder(source) + entry = data.putIfAbsent(source, new NameRef(source, maybe)) + if (entry eq null) + res = maybe + } else { + res = entry.get() + if (res eq null) { + val maybe = builder(source) + if (data.replace(source, entry, new NameRef(source, maybe))) + res = maybe + else { + res = null + entry = data.get(source) + } + } + } + } while (res eq null) + res + } +} diff --git a/src/reflect/scala/reflect/internal/names/WeakFixedSizeConcurrentNodeInterner.scala b/src/reflect/scala/reflect/internal/names/WeakFixedSizeConcurrentNodeInterner.scala new file mode 100644 index 00000000000..022f0f07402 --- /dev/null +++ b/src/reflect/scala/reflect/internal/names/WeakFixedSizeConcurrentNodeInterner.scala @@ -0,0 +1,351 @@ +package scala.reflect.internal.names + +import java.util.concurrent.atomic.{AtomicInteger, AtomicReferenceArray, AtomicReferenceFieldUpdater} +import java.lang.ref._ +import java.util + +/** + * a Node Interner that allows for concurrent access + * references to Names are held weakly + * + * Data table has a fixed entry size + * + * Data table head is only appended with CAS + * a data row is trimmed when traversed, with CAS + * + * @param createName + * @tparam T + */ +sealed abstract class AbstractWeakFixedSizeConcurrentNodeInterner[T >: Null <: NameBase](createName: String => T) extends AbstractNodeInterner[T](createName) { + type N = WeakConcurrentNode[T] + + def createNode(name: T, next: WeakConcurrentNode[T], bucket: Int): N + + def find(key: String): T = { + val hash = key.hashCode + val improved = improveHash(hash) + var oldTail: N = null + + var name: T = null + val bucket = improved & mask + do { + val head = data.get(bucket) + if (head ne null) + name = WeakConcurrentNode.findNoTrim(head, key, hash, oldTail) + if (name eq null) { + // minor optimisation - we can skip this tail if we have to retry + // if we came to the end of the chain of nodes we dont need to search the same tail if we fail and try again + oldTail = head + name = createName(key) + val newNode = createNode(name, head, bucket) + if (data.compareAndSet(bucket, head, newNode)) { + size_.incrementAndGet() + } else name = null + } + } while (name eq null) + name + } + + //only public until we select an implementation + def find1(chars: Array[Char], start: Int, count: Int): T = { + val hash = hashLikeString(chars, start, count) + val improved = improveHash(hash) + var oldTail: N = null + + var name: T = null + val bucket = improved & mask + val head = data.get(bucket) + if (head ne null) + name = WeakConcurrentNode.findNoTrimC1(head, chars, start, count, hash, oldTail) + if (name eq null) { + // minor optimisation - we can skip this tail if we have to retry + // if we came to the end of the chain of nodes we dont need to search the same tail if we fail and try again + oldTail = head + val string = new String(chars, start, count) + name = createName(string) + val newNode = createNode(name, head, bucket) + if (data.compareAndSet(bucket, head, newNode)) { + size_.incrementAndGet() + } else name = find(string) + } + name + } + + //only public until we select an implementation + def find2(chars: Array[Char], start: Int, count: Int): T = { + val hash = hashLikeString(chars, start, count) + val improved = improveHash(hash) + var oldTail: N = null + + var name: T = null + val bucket = improved & mask + val head = data.get(bucket) + if (head ne null) + name = WeakConcurrentNode.findNoTrimC2(head, chars, start, count, hash, oldTail) + if (name eq null) { + // minor optimisation - we can skip this tail if we have to retry + // if we came to the end of the chain of nodes we dont need to search the same tail if we fail and try again + oldTail = head + val string = new String(chars, start, count) + name = createName(string) + val newNode = createNode(name, head, bucket) + if (data.compareAndSet(bucket, head, newNode)) { + size_.incrementAndGet() + } else name = find(string) + } + name + } + + //only public until we select an implementation + def find3(chars: Array[Char], start: Int, count: Int): T = { + val hash = hashLikeString(chars, start, count) + val improved = improveHash(hash) + var oldTail: N = null + + var name: T = null + val bucket = improved & mask + val head = data.get(bucket) + if (head ne null) + name = WeakConcurrentNode.findNoTrimC3(head, chars, start, count, hash, oldTail) + if (name eq null) { + // minor optimisation - we can skip this tail if we have to retry + // if we came to the end of the chain of nodes we dont need to search the same tail if we fail and try again + oldTail = head + val string = new String(chars, start, count) + name = createName(string) + val newNode = createNode(name, head, bucket) + if (data.compareAndSet(bucket, head, newNode)) { + size_.incrementAndGet() + } else name = find(string) + } + name + } + + private val size_ = new AtomicInteger() + + override def size: Int = size_.get + + override def getExistingImpl(key: String): T = { + val hash = key.hashCode + val improved = improveHash(hash) + + val head = data.get(improved & (data.length - 1)) + WeakConcurrentNode.findNoTrim(head, key, hash, null) + + } + + protected final def trimImpl(index: Int): Unit = { + var head: N = null + var value: AnyRef = null + do { + head = data.get(index) + if (head eq null) + value = this + else { + value = head.name + if ((value eq null) && data.compareAndSet(index, head, head.next)) + size_.decrementAndGet() + } + } while (value eq null) + + size_.addAndGet(-WeakConcurrentNode.trimCollected(head)) + } + + @inline final val data = new AtomicReferenceArray[N](1 << 16) + + @inline final private def mask = (1 << 16) - 1 +} + +class WeakFixedSizeNoAutoTrimConcurrentNodeInterner[T >: Null <: NameBase](createName: String => T) extends AbstractWeakFixedSizeConcurrentNodeInterner(createName) { + override def createNode(name: T, next: WeakConcurrentNode[T], bucket: Int) = new WeakConcurrentNode[T](new WeakReference(name), next) + + def trim() = synchronized { + for (index <- 0 until data.length) + trimImpl(index) + } +} + +class WeakFixedSizeAutoTrimConcurrentNodeInterner[T >: Null <: NameBase](createName: String => T) extends AbstractWeakFixedSizeConcurrentNodeInterner(createName) { + private val queue = new ReferenceQueue[T] + + override def createNode(name: T, next: WeakConcurrentNode[T], bucket: Int) = new WeakConcurrentNode[T](new BucketReference(name, queue, bucket), next) + + private class BucketReference(name: T, queue: ReferenceQueue[T], val bucket: Int) extends WeakReference[T](name, queue) + + override def size: Int = { + trimCollected() + super.size + } + + override def find(key: String): T = { + trimCollected() + super.find(key) + } + + @inline private def trimCollected(): Unit = { + val next = queue.poll() + if (next ne null) { + reallyTrim(next) + } + } + + private def reallyTrim(first: Reference[_]): Unit = { + var next = first + val buckets = new util.BitSet(data.length) + do { + buckets.set(next.asInstanceOf[BucketReference].bucket) + next = queue.poll() + } while (next ne null) + + queue.synchronized { + var index = buckets.nextSetBit(0) + while (index != -1) { + trimImpl(index) + index = buckets.nextSetBit(index + 1) + } + } + } +} + +object AbstractUnsafeWeakFixedSizeConcurrentNodeInterner { + + private[this] val unsafe = { + val f = classOf[sun.misc.Unsafe].getDeclaredField("theUnsafe") + f.setAccessible(true) + f.get(null).asInstanceOf[sun.misc.Unsafe] + } + private[this] val base = unsafe.arrayBaseOffset(classOf[Array[WeakConcurrentNode[_]]]) + private[this] val shift = { + val scale = unsafe.arrayIndexScale(classOf[Array[WeakConcurrentNode[_]]]) + if (Integer.bitCount(scale) != 1) + throw new Error("data type scale not a power of two") + 31 - Integer.numberOfLeadingZeros(scale) + } + @inline private[this] def offset(bucket: Int) = + (bucket.toLong << shift) + base + + def compareAndSet[T >: Null <: NameBase](data: Array[WeakConcurrentNode[T]], bucket: Int, oldNode: WeakConcurrentNode[T], newNode: WeakConcurrentNode[T]): Boolean = { + unsafe.compareAndSwapObject(data, offset(bucket), oldNode, newNode) + } + + def getVolatile[T >: Null <: NameBase](data: Array[WeakConcurrentNode[T]], bucket: Int): WeakConcurrentNode[T] = { + unsafe.getObjectVolatile(data, offset(bucket)).asInstanceOf[WeakConcurrentNode[T]] + } +} + +sealed abstract class AbstractUnsafeWeakFixedSizeConcurrentNodeInterner[T >: Null <: NameBase](createName: String => T) extends AbstractNodeInterner[T](createName) { + + import AbstractUnsafeWeakFixedSizeConcurrentNodeInterner._ + + type N = WeakConcurrentNode[T] + + def createNode(name: T, next: WeakConcurrentNode[T], bucket: Int): N + + def find(key: String): T = { + val hash = key.hashCode + val improved = improveHash(hash) + var oldTail: N = null + + var name: T = null + val bucket = improved & mask + do { + val head = getVolatile(data, bucket) + if (head ne null) + name = WeakConcurrentNode.findNoTrim(head, key, hash, oldTail) + if (name eq null) { + // minor optimisation - we can skip this tail if we have to retry + // if we came to the end of the chain of nodes we dont need to search the same tail if we fail and try again + oldTail = head + name = createName(key) + val newNode = createNode(name, head, bucket) + if (compareAndSet(data, bucket, head, newNode)) { + size_.incrementAndGet() + } else name = null + } + } while (name eq null) + name + } + + private val size_ = new AtomicInteger() + + override def size: Int = size_.get + + override def getExistingImpl(key: String): T = { + val hash = key.hashCode + val improved = improveHash(hash) + + val head = getVolatile(data, improved & (data.length - 1)) + WeakConcurrentNode.findNoTrim(head, key, hash, null) + + } + + protected final def trimImpl(index: Int): Unit = { + var head: N = null + var value: AnyRef = null + do { + head = getVolatile(data, index) + if (head eq null) + value = this + else { + value = head.name + if ((value eq null) && compareAndSet(data, index, head, head.next)) + size_.decrementAndGet() + } + } while (value eq null) + + size_.addAndGet(-WeakConcurrentNode.trimCollected(head)) + } + + @inline final val data = new Array[N](1 << 16) + + @inline final private def mask = (1 << 16) - 1 +} +class UnsafeWeakFixedSizeNoAutoTrimConcurrentNodeInterner[T >: Null <: NameBase](createName: String => T) extends AbstractUnsafeWeakFixedSizeConcurrentNodeInterner(createName) { + override def createNode(name: T, next: WeakConcurrentNode[T], bucket: Int) = new WeakConcurrentNode[T](new WeakReference(name), next) + + def trim() = synchronized { + for (index <- 0 until data.length) + trimImpl(index) + } +} +class UnsafeWeakFixedSizeAutoTrimConcurrentNodeInterner[T >: Null <: NameBase](createName: String => T) extends AbstractUnsafeWeakFixedSizeConcurrentNodeInterner(createName) { + private val queue = new ReferenceQueue[T] + + override def createNode(name: T, next: WeakConcurrentNode[T], bucket: Int) = new WeakConcurrentNode[T](new BucketReference(name, queue, bucket), next) + + private class BucketReference(name: T, queue: ReferenceQueue[T], val bucket: Int) extends WeakReference[T](name, queue) + + override def size: Int = { + trimCollected() + super.size + } + + override def find(key: String): T = { + trimCollected() + super.find(key) + } + + @inline private def trimCollected(): Unit = { + val next = queue.poll() + if (next ne null) { + reallyTrim(next) + } + } + + private def reallyTrim(first: Reference[_]): Unit = { + var next = first + val buckets = new util.BitSet(data.length) + do { + buckets.set(next.asInstanceOf[BucketReference].bucket) + next = queue.poll() + } while (next ne null) + + queue.synchronized { + var index = buckets.nextSetBit(0) + while (index != -1) { + trimImpl(index) + index = buckets.nextSetBit(index + 1) + } + } + } +} diff --git a/src/reflect/scala/reflect/internal/names/WeakMapNameTable.scala b/src/reflect/scala/reflect/internal/names/WeakMapNameTable.scala new file mode 100644 index 00000000000..4017ab19394 --- /dev/null +++ b/src/reflect/scala/reflect/internal/names/WeakMapNameTable.scala @@ -0,0 +1,49 @@ +package scala.reflect.internal.names + +import java.lang.ref.{Reference, ReferenceQueue, WeakReference} + +import scala.annotation.tailrec +import scala.collection.mutable + +final class WeakMapNameTable[T <: AnyRef](builder: (String) => T) extends NameTable[T] { + class NameRef(val key:String, name:T) extends WeakReference(name, queue) + + private val data = new mutable.AnyRefMap[String, NameRef] + private val queue = new ReferenceQueue[T] + + + @tailrec def processQueue(): Unit = { + val next = queue.poll().asInstanceOf[NameRef] + if (next ne null) { + data.getOrNull(next.key) match { + case null => + case ref => + if (ref.get() eq null) + data.remove(ref.key) + } + processQueue() + } + } + + override def size: Int = { + processQueue() + data.size + } + + def find(source: String): T = { + processQueue() + data.getOrNull(source) match { + case null => + val res = builder(source) + data(source) = new NameRef(source, res) + res + case ref => + var res = ref.get() + if (res eq null) { + res = builder(source) + data(source) = new NameRef(source, res) + } + res + } + } +} diff --git a/test/benchmarks/src/main/scala/scala/reflect/ExistingNameTable.scala b/test/benchmarks/src/main/scala/scala/reflect/ExistingNameTable.scala new file mode 100644 index 00000000000..bf2a61d7998 --- /dev/null +++ b/test/benchmarks/src/main/scala/scala/reflect/ExistingNameTable.scala @@ -0,0 +1,17 @@ +package scala.reflect + +import scala.reflect.internal.Names +import scala.reflect.internal.names.{NameBase, NameTable} + +class ExistingSynchronizedNameTable extends ExistingNameTable{ + override def synchronizeNames: Boolean = true +} +class ExistingNameTable extends NameTable[AnyRef] with Names{ + override def size: Int = -1 + + override def find(source: String) = newTermName(source.toCharArray(), 0, source.length(), null) + + + override def find(chars: Array[Char], start: Int, count: Int) = newTermName(chars, start, count) + +} diff --git a/test/benchmarks/src/main/scala/scala/reflect/NamesBenchmark.scala b/test/benchmarks/src/main/scala/scala/reflect/NamesBenchmark.scala new file mode 100644 index 00000000000..950794c74f6 --- /dev/null +++ b/test/benchmarks/src/main/scala/scala/reflect/NamesBenchmark.scala @@ -0,0 +1,114 @@ +package scala.reflect + +import java.util.concurrent.TimeUnit + +import org.openjdk.jmh.annotations._ +import org.openjdk.jmh.infra.Blackhole + +import scala.reflect.internal.names._ + +@BenchmarkMode(Array(Mode.AverageTime)) +@Fork(2) +@Warmup(iterations = 10) +@Measurement(iterations = 10) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@State(Scope.Benchmark) +abstract class AbstractNamesBenchmark { + var names: Array[String] = null + var namesChars: Array[Array[Char]] = null + + @Param(Array("0", "10", "100","1000", "10000", "100000")) + var arraySize: Int = _ + + def testType: String + + var fullNameTable: NameTable[_] = _ + var newNameTable: () => NameTable[_] = _ + + @Setup def setup(): Unit = { + names = Array.tabulate(arraySize)(i => s"name$i") + namesChars = names map (_.toArray) + newNameTable = testType match { + case "Existing" => () => new ExistingNameTable + case "ExistingSynchronized" => () => new ExistingSynchronizedNameTable + case "MapNameTable" => () => new MapNameTable (new NameBase(_)) + case "ConcurrentMapNameTable" => () => new ConcurrentMapNameTable(new NameBase(_)) + case "WeakConcurrentMapNameTable" => () => new WeakConcurrentMapNameTable(new NameBase(_)) + case "StrongConcurrentNodeInterner" => () => new StrongConcurrentNodeInterner(new NameBase(_)) + case "WeakFixedSizeNoAutoTrimConcurrentNodeInterner" => () => new WeakFixedSizeNoAutoTrimConcurrentNodeInterner(new NameBase(_)) + case "WeakFixedSizeAutoTrimConcurrentNodeInterner" => () => new WeakFixedSizeAutoTrimConcurrentNodeInterner(new NameBase(_)) + case "Find0" => () => new WeakFixedSizeNoAutoTrimConcurrentNodeInterner(new NameBase(_)) { + override def find(chars: Array[Char], start: Int, count: Int) = super.find(chars, start, count) + } + case "Find1" => () => new WeakFixedSizeNoAutoTrimConcurrentNodeInterner(new NameBase(_)) { + override def find(chars: Array[Char], start: Int, count: Int) = super.find1(chars, start, count) + } + case "Find2" => () => new WeakFixedSizeNoAutoTrimConcurrentNodeInterner(new NameBase(_)) { + override def find(chars: Array[Char], start: Int, count: Int) = super.find2(chars, start, count) + } + case "Find3" => () => new WeakFixedSizeNoAutoTrimConcurrentNodeInterner(new NameBase(_)) { + override def find(chars: Array[Char], start: Int, count: Int) = super.find3(chars, start, count) + } + case "Safe" => () => new WeakFixedSizeNoAutoTrimConcurrentNodeInterner(new NameBase(_)) { + override def find(chars: Array[Char], start: Int, count: Int) = super.find(chars, start, count) + } + case "Unsafe" => () => new UnsafeWeakFixedSizeNoAutoTrimConcurrentNodeInterner(new NameBase(_)) { + override def find(chars: Array[Char], start: Int, count: Int) = super.find(chars, start, count) + } + } + fullNameTable = newNameTable() + names foreach {fullNameTable.find(_)} + } + + @Benchmark def findOldString(bh: Blackhole): Any = { + val nameTable: NameTable[_] = fullNameTable + var i = 0 + while (i < arraySize) { + bh.consume(nameTable.find(names(i))) + i += 1 + } + } + @Benchmark def findOldChars(bh: Blackhole): Any = { + val nameTable: NameTable[_] = fullNameTable + var i = 0 + while (i < arraySize) { + val c= namesChars(i) + bh.consume(nameTable.find(c,0,c.length)) + i += 1 + } + } +} +@Threads(1) +class NamesBenchmark extends AbstractNamesBenchmark { + @Param(Array("Existing", "ExistingSynchronized", "MapNameTable", "ConcurrentMapNameTable", + "WeakConcurrentMapNameTable", "StrongConcurrentNodeInterner", "WeakFixedSizeNoAutoTrimConcurrentNodeInterner", + "WeakFixedSizeAutoTrimConcurrentNodeInterner", "Find0", "Find1", "Find2", "Find3", "Safe", "Unsafe")) + var testImpl: String = _ + override def testType = testImpl + @Benchmark def findNewString(bh: Blackhole): Any = { + val nameTable = newNameTable() + var i = 0 + while (i < arraySize) { + bh.consume(nameTable.find(names(i))) + i += 1 + } + } + + @Benchmark def findNewChars(bh: Blackhole): Any = { + val nameTable = newNameTable() + var i = 0 + while (i < arraySize) { + val c= namesChars(i) + bh.consume(nameTable.find(c,0,c.length)) + i += 1 + } + } +} +@Threads(4) +class NamesBenchmarkMT extends AbstractNamesBenchmark { + @Param(Array("ExistingSynchronized", "ConcurrentMapNameTable", "WeakConcurrentMapNameTable", + "StrongConcurrentNodeInterner", "WeakFixedSizeNoAutoTrimConcurrentNodeInterner", + "WeakFixedSizeAutoTrimConcurrentNodeInterner", "Find0", "Find1", "Find2", "Find3", "Safe", "Unsafe")) + var testImpl: String = _ + override def testType = testImpl +} \ No newline at end of file diff --git a/test/junit/scala/reflect/internal/names/ConcurrentNamesTest.scala b/test/junit/scala/reflect/internal/names/ConcurrentNamesTest.scala new file mode 100644 index 00000000000..2571ac973bf --- /dev/null +++ b/test/junit/scala/reflect/internal/names/ConcurrentNamesTest.scala @@ -0,0 +1,33 @@ +package scala.reflect.internal.names + +import java.util.concurrent.{CyclicBarrier, Executors, TimeUnit} + +import org.junit.Assert.assertSame +import org.junit.Test + +import scala.concurrent.duration.Duration +import scala.concurrent.{Await, ExecutionContext, Future} + +trait ConcurrentNamesTest { + self: ExtendedNameTest => + @Test def concurrentLookup: Unit = { + val threads = 20 + val barrier = new CyclicBarrier(threads) + val pool = Executors.newCachedThreadPool() + implicit val context = ExecutionContext.fromExecutor(pool) + try { + val futures = for (thread <- 0 until threads) yield { + Future{ + barrier.await() + lookupInRandom + } + } + val results = futures map {Await.result(_, Duration(1, TimeUnit.MINUTES))} + val first = results(0) + for (result <- results; + i <- 0 until sources.length) { + assertSame(first(i), result(i)) + } + } finally pool.shutdownNow() + } +} diff --git a/test/junit/scala/reflect/internal/names/ExtendedNameTest.scala b/test/junit/scala/reflect/internal/names/ExtendedNameTest.scala new file mode 100644 index 00000000000..84c484396f1 --- /dev/null +++ b/test/junit/scala/reflect/internal/names/ExtendedNameTest.scala @@ -0,0 +1,127 @@ +package scala.reflect.internal.names + +import java.util +import java.util.concurrent.{CyclicBarrier, Executors, TimeUnit} + +import org.junit.Assert._ +import org.junit.Test + +import scala.concurrent.duration.Duration +import scala.concurrent.{Await, ExecutionContext, Future} +import scala.util.Random + +abstract class ExtendedNameTest { + type T <: AnyRef + val nameTable: NameTable[T] + def newTermName(s: String) = { + nameTable.find(s) + } + def newTermName(c:Array[Char], off:Int, len:Int) = { + nameTable.find(c,off,len) + } + lazy val sources = Array.tabulate(10000)( i => s"n$i") + + @Test + def checkSimple(): Unit = { + assertEquals(0, nameTable.size) + val n1 = newTermName("xx") + assertEquals(1, nameTable.size) + + val n2 = newTermName("xx") + assertEquals(1, nameTable.size) + assertSame(n1,n2) + + val n3 = newTermName(new String("xx")) + assertEquals(1, nameTable.size) + assertSame(n1,n3) + assertEquals(n1,n3) + } + @Test def checkByCharsAndByString() { + assertEquals(0, nameTable.size) + val byString = sources map newTermName + + val byChars = sources map { + s => + newTermName(s.toCharArray, 0, s.length) + } + val byChars2 = sources map { + s => + newTermName("abc".toCharArray ++ s.toCharArray ++ "xyz", 3, s.length) + } + assertEquals(sources.length, nameTable.size) + for (i <- 0 until sources.length) { + assertSame(byString(i), byChars(i)) + assertSame(byString(i), byChars2(i)) + } + + } + @Test def checkByCharsAndByString2() { + assertEquals(0, nameTable.size) + //same as previous but do the chars first + val byChars = sources map { + s => + newTermName(s.toCharArray, 0, s.length) + } + val byString = sources map newTermName + + val byChars2 = sources map { + s => + newTermName("abc".toCharArray ++ s.toCharArray ++ "xyz", 3, s.length) + } + assertEquals(sources.length, nameTable.size) + for (i <- 0 until sources.length) { + assertSame(byString(i), byChars(i)) + assertSame(byString(i), byChars2(i)) + } + + } + @Test + def checkSources1(): Unit = { + assertEquals(0, nameTable.size) + val terms = sources map newTermName + + assertEquals(sources.length, nameTable.size) + val different = terms.toSet + assertEquals(sources.length, different.size) + + val shuffled: List[Int] = Random.shuffle(List.tabulate(sources.length)(i => i)) + + shuffled.foreach{ + i => + val again = newTermName(new String(sources(i))) + assertSame(terms(i), again) + assertEquals(terms(i), again) + } + } + + @Test + def checkSources2(): Unit = { + lookupInRandom + } + def lookupInRandom = { + assertEquals(0, nameTable.size) + val random = new Random() + val size = sources.size + val all = new Array[AnyRef](sources.length) + for (count <- 0 until size * 10) { + val i = random.nextInt(size) + val term = newTermName(new String(sources(i))) + if (null eq all(i) ) { + all(i) = term + } else { + assertSame(all(i), term) + } + } + + for (i <- 0 until size) { + val term = newTermName(new String(sources(i))) + if (null eq all(i) ) { + all(i) = term + } else { + assertSame(all(i), term) + } + } + all + } +} + diff --git a/test/junit/scala/reflect/internal/names/NamesImplTest.scala b/test/junit/scala/reflect/internal/names/NamesImplTest.scala new file mode 100644 index 00000000000..ef0a47c2427 --- /dev/null +++ b/test/junit/scala/reflect/internal/names/NamesImplTest.scala @@ -0,0 +1,67 @@ +package scala.reflect.internal.names + +case class Term(s:String) extends NameBase(s) + +class MapNamesTabelTest extends ExtendedNameTest { + override type T = Term + override val nameTable: NameTable[T] = new MapNameTable[Term](Term.apply) +} +class ConcurrentMapNamesTabelTest extends ExtendedNameTest { + override type T = Term + override val nameTable: NameTable[T] = new ConcurrentMapNameTable[Term](Term.apply) +} +class StrongConcurrentNodeInternerTest extends ExtendedNameTest with ConcurrentNamesTest { + override type T = Term + override val nameTable: NameTable[T] = new StrongConcurrentNodeInterner[Term](Term.apply) +} +class WeakMapNameTableTest extends ExtendedNameTest with WeakNamesTest { + override type T = Term + override val nameTable: NameTable[T] = new WeakMapNameTable[Term](Term.apply) +} +class WeakConcurrentMapNameTableTest extends ExtendedNameTest with WeakNamesTest { + override type T = Term + override val nameTable: NameTable[T] = new WeakConcurrentMapNameTable[Term](Term.apply) +} +class WeakFixedSizeNoAutoTrimConcurrentNodeInternerTest extends ExtendedNameTest with WeakNamesTest { + override type T = Term + override val nameTable: WeakFixedSizeNoAutoTrimConcurrentNodeInterner[T] = new WeakFixedSizeNoAutoTrimConcurrentNodeInterner[Term](Term.apply) + + override def cleanupIfNeeded(): Unit = nameTable.trim() +} +class WeakFixedSizeAutoTrimConcurrentNodeInternerTest extends ExtendedNameTest with WeakNamesTest { + override type T = Term + override val nameTable: NameTable[T] = new WeakFixedSizeAutoTrimConcurrentNodeInterner[Term](Term.apply) + +} +class Find0 extends ExtendedNameTest { + override type T = Term + override val nameTable: NameTable[T] = new WeakFixedSizeAutoTrimConcurrentNodeInterner[Term](Term.apply) { + override def find(chars: Array[Char], start: Int, count: Int): Term = super.find(chars, start, count) + } +} +class Find1 extends ExtendedNameTest { + override type T = Term + override val nameTable: NameTable[T] = new WeakFixedSizeAutoTrimConcurrentNodeInterner[Term](Term.apply) { + override def find(chars: Array[Char], start: Int, count: Int): Term = super.find1(chars, start, count) + } +} +class Find2 extends ExtendedNameTest { + override type T = Term + override val nameTable: NameTable[T] = new WeakFixedSizeAutoTrimConcurrentNodeInterner[Term](Term.apply) { + override def find(chars: Array[Char], start: Int, count: Int): Term = super.find2(chars, start, count) + } +} +class Find3 extends ExtendedNameTest { + override type T = Term + override val nameTable: NameTable[T] = new WeakFixedSizeAutoTrimConcurrentNodeInterner[Term](Term.apply) { + override def find(chars: Array[Char], start: Int, count: Int): Term = super.find3(chars, start, count) + } +} +class Safe extends ExtendedNameTest { + override type T = Term + override val nameTable: NameTable[T] = new WeakFixedSizeAutoTrimConcurrentNodeInterner[Term](Term.apply) +} +class Unsafe extends ExtendedNameTest { + override type T = Term + override val nameTable: NameTable[T] = new UnsafeWeakFixedSizeAutoTrimConcurrentNodeInterner[Term](Term.apply) +} diff --git a/test/junit/scala/reflect/internal/names/WeakNamesTest.scala b/test/junit/scala/reflect/internal/names/WeakNamesTest.scala new file mode 100644 index 00000000000..e42751286ea --- /dev/null +++ b/test/junit/scala/reflect/internal/names/WeakNamesTest.scala @@ -0,0 +1,88 @@ +package scala.reflect.internal.names + +import java.util + +import org.junit._ +import Assert._ + +import java.lang.ref.WeakReference + +trait WeakNamesTest { + self: ExtendedNameTest => + + val weakSources = Vector.tabulate(100)(i => s"weak${i}") + + private def gcHard(): Unit = { + for (i <- 1 to 3) { + System.gc() + System.runFinalization() + Thread.sleep(1) + } + } + private def gcHard(completeWhenClear: WeakReference[T]): Unit = { + for (i <- 1 to 10 if (completeWhenClear.get ne null)) { + System.gc() + System.runFinalization() + Thread.sleep(1) + } + } + + @Test def simpleWeak: Unit = { + val size = weakSources.length + val strongRefs = new Array[AnyRef](size) + for (i <- 0 until size) { + val name = newTermName(weakSources(i)) + assertNotNull(name) + strongRefs(i) = name + } + assertEquals(size, nameTable.size) + gcHard + assertEquals(size, nameTable.size) + gcHard + + util.Arrays.fill(strongRefs, null) + assertEquals(size, nameTable.size) + } + + // for colletction that need explicit trimming after GC, trimm them + def cleanupIfNeeded() = {} + + @Test def slidingWeak: Unit = { + val size = weakSources.length + val weakRefs = new Array[WeakReference[T]](size) + val strongRefs = new Array[AnyRef](size) + val lag = 10 + for (i <- 0 until lag) { + val name = newTermName(weakSources(i)) + assertNotNull(name) + weakRefs(i) = new WeakReference(name) + strongRefs(i) = name + } + assertEquals(lag, nameTable.size) + gcHard() + assertEquals(lag, nameTable.size) + for (tail <- 0 until size) { + val head = tail + lag + if (head < size) { + val name = newTermName(weakSources(head)) + assertNotNull(name) + weakRefs(head) = new WeakReference(name) + strongRefs(head) = name + assertEquals(lag + 1, nameTable.size) + + gcHard() + assertEquals(lag + 1, nameTable.size) + assertNotNull(weakRefs(tail).get) + assertEquals(lag + 1, nameTable.size) + } + assertNotNull(strongRefs(tail)) + strongRefs(tail) = null + gcHard(weakRefs(tail)) + assertNull(s"still a reference to ${weakSources(tail)}", weakRefs(tail).get) + cleanupIfNeeded() + assertEquals(s"at tail $tail, head $head, size = ${nameTable.size}", math.min(head, size - 1) - tail, nameTable.size) + } + assertEquals(0, nameTable.size) + } + +} From 69f806d8f9385a08df0eb30cba18d118efe901a7 Mon Sep 17 00:00:00 2001 From: Mike Skells Date: Thu, 20 Jun 2019 23:19:47 +0100 Subject: [PATCH 3/7] misplaced assert - only valid for non concurrent tests --- test/junit/scala/reflect/internal/names/ExtendedNameTest.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/junit/scala/reflect/internal/names/ExtendedNameTest.scala b/test/junit/scala/reflect/internal/names/ExtendedNameTest.scala index 84c484396f1..38cf4521137 100644 --- a/test/junit/scala/reflect/internal/names/ExtendedNameTest.scala +++ b/test/junit/scala/reflect/internal/names/ExtendedNameTest.scala @@ -96,10 +96,10 @@ abstract class ExtendedNameTest { @Test def checkSources2(): Unit = { + assertEquals(0, nameTable.size) lookupInRandom } def lookupInRandom = { - assertEquals(0, nameTable.size) val random = new Random() val size = sources.size val all = new Array[AnyRef](sources.length) From 9942f019cda588c00c2f763add00fb71726d8fde Mon Sep 17 00:00:00 2001 From: Mike Skells Date: Fri, 21 Jun 2019 17:59:07 +0100 Subject: [PATCH 4/7] simplify some code remove unused local and optimise locals uasge add some more tests add some more benchmarks --- .../scala/reflect/internal/names/Node.scala | 271 +++++++++--------- .../names/StrongConcurrentNodeInterner.scala | 10 +- .../WeakFixedSizeConcurrentNodeInterner.scala | 83 ++++-- .../scala/scala/reflect/NamesBenchmark.scala | 10 +- .../internal/names/NamesImplTest.scala | 12 + 5 files changed, 221 insertions(+), 165 deletions(-) diff --git a/src/reflect/scala/reflect/internal/names/Node.scala b/src/reflect/scala/reflect/internal/names/Node.scala index 468ae9c21b2..b88845a570d 100644 --- a/src/reflect/scala/reflect/internal/names/Node.scala +++ b/src/reflect/scala/reflect/internal/names/Node.scala @@ -62,6 +62,7 @@ object WeakNode { } } } + final class WeakNode[N >: Null <: NameBase](k: N, @volatile var next: WeakNode[N]) extends Node[N] { private val ref = new WeakReference(k) @@ -71,6 +72,7 @@ final class WeakNode[N >: Null <: NameBase](k: N, @volatile var next: WeakNode[N } def name = ref.get() + def isDefined = ref.get() ne null def hash: Int = { @@ -82,133 +84,128 @@ final class WeakNode[N >: Null <: NameBase](k: N, @volatile var next: WeakNode[N object WeakConcurrentNode { private val nextAccess: AtomicReferenceFieldUpdater[WeakConcurrentNode[_], WeakConcurrentNode[_]] = new WeakConcurrentNode[NameBase](null, null).nextAccess - private def nextAccessN[N>: Null <:NameBase ]: AtomicReferenceFieldUpdater[WeakConcurrentNode[N], WeakConcurrentNode[N]] = nextAccess.asInstanceOf[AtomicReferenceFieldUpdater[WeakConcurrentNode[N], WeakConcurrentNode[N]]] - private def casNext[N>: Null <:NameBase ](prev: WeakConcurrentNode[N], expectedNext: WeakConcurrentNode[N], updateNext: WeakConcurrentNode[N]): Boolean = { + private def nextAccessN[N >: Null <: NameBase]: AtomicReferenceFieldUpdater[WeakConcurrentNode[N], WeakConcurrentNode[N]] = nextAccess.asInstanceOf[AtomicReferenceFieldUpdater[WeakConcurrentNode[N], WeakConcurrentNode[N]]] + + private def casNext[N >: Null <: NameBase](prev: WeakConcurrentNode[N], expectedNext: WeakConcurrentNode[N], updateNext: WeakConcurrentNode[N]): Boolean = { nextAccess.compareAndSet(prev, expectedNext, updateNext) } + private val builderCache = new ThreadLocal[java.lang.StringBuilder] { - override def initialValue()= new java.lang.StringBuilder + override def initialValue() = new java.lang.StringBuilder } - final def findNoTrimC1[N >: Null <: NameBase](start: WeakConcurrentNode[N], chars: Array[Char], charOffset: Int, charCount: Int, hash: Int, end: Node[N]): N = { - def equalsAsString(nameId:String): Boolean = { + final def findNoTrimC1[N >: Null <: NameBase](start: WeakConcurrentNode[N], chars: Array[Char], charOffset: Int, charCount: Int, hash: Int): N = { + def equalsAsString(nameId: String): Boolean = { nameId.hashCode() == hash && nameId.length == charCount && { var offset = 0 var equal = true while (offset < charCount && equal) { - equal = chars(charOffset+offset) == nameId.charAt(offset) - offset +=1 + equal = chars(charOffset + offset) == nameId.charAt(offset) + offset += 1 } equal } } - if (start eq end) - null - else { - var current = start - var prev: WeakConcurrentNode[N] = null - - var currentName: N = null - var found: N = null - - do { - currentName = current.name - if ((currentName ne null) && equalsAsString(currentName.id)) - found = currentName - else { - prev = current - current = current.next - } - } while ((found eq null) && (current ne null) && (current ne end)) - found - } + var current = start + var prev: WeakConcurrentNode[N] = null + + var currentName: N = null + var found: N = null + + do { + currentName = current.name + if ((currentName ne null) && equalsAsString(currentName.id)) + found = currentName + else { + prev = current + current = current.next + } + } while ((found eq null) && (current ne null)) + found } - final def findNoTrimC2[N >: Null <: NameBase](start: WeakConcurrentNode[N], chars: Array[Char], charOffset: Int, charCount: Int, hash: Int, end: Node[N]): N = { - var seq:java.lang.StringBuilder = null - if (start eq end) - null - else { - var current = start - var prev: WeakConcurrentNode[N] = null - var currentName: N = null - var found: N = null + final def findNoTrimC2[N >: Null <: NameBase](start: WeakConcurrentNode[N], chars: Array[Char], charOffset: Int, charCount: Int, hash: Int): N = { + var seq: java.lang.StringBuilder = null + var current = start + var prev: WeakConcurrentNode[N] = null - do { - currentName = current.name - if ((currentName ne null) && currentName.id.hashCode() == hash && currentName.id.length == charCount && { - if (seq == null) { - seq = new java.lang.StringBuilder() - seq.append(chars, charOffset, charCount) - } - currentName.id.contentEquals(seq) - }) - found = currentName - else { - prev = current - current = current.next - } - } while ((found eq null) && (current ne null) && (current ne end)) - found - } + var currentName: N = null + var found: N = null + do { + currentName = current.name + if ((currentName ne null) && currentName.id.hashCode() == hash && currentName.id.length == charCount && { + if (seq == null) { + seq = new java.lang.StringBuilder() + seq.append(chars, charOffset, charCount) + } + currentName.id.contentEquals(seq) + }) + found = currentName + else { + prev = current + current = current.next + } + } while ((found eq null) && (current ne null)) + found } - final def findNoTrimC3[N >: Null <: NameBase](start: WeakConcurrentNode[N], chars: Array[Char], charOffset: Int, charCount: Int, hash: Int, end: Node[N]): N = { - var seq:java.lang.StringBuilder = null - if (start eq end) - null - else { - var current = start - var prev: WeakConcurrentNode[N] = null + final def findNoTrimC3[N >: Null <: NameBase](start: WeakConcurrentNode[N], chars: Array[Char], charOffset: Int, charCount: Int, hash: Int): N = { + var seq: java.lang.StringBuilder = null + var current = start + var prev: WeakConcurrentNode[N] = null - var currentName: N = null - var found: N = null + var currentName: N = null + var found: N = null - do { - currentName = current.name - if ((currentName ne null) && currentName.id.hashCode() == hash && currentName.id.length == charCount && { - if (seq == null) { - seq = builderCache.get() - seq.append(chars, charOffset, charCount) - } - currentName.id.contentEquals(seq) - }) - found = currentName - else { - prev = current - current = current.next + do { + currentName = current.name + if ((currentName ne null) && currentName.id.hashCode() == hash && currentName.id.length == charCount && { + if (seq == null) { + seq = builderCache.get() + seq.append(chars, charOffset, charCount) } - } while ((found eq null) && (current ne null) && (current ne end)) - if (seq ne null) - seq.setLength(0) - found - } - + currentName.id.contentEquals(seq) + }) + found = currentName + else { + prev = current + current = current.next + } + } while ((found eq null) && (current ne null)) + if (seq ne null) + seq.setLength(0) + found } + final def findNoTrim[N >: Null <: NameBase](start: WeakConcurrentNode[N], key: String, hash: Int, end: Node[N]): N = { - if (start eq end) - null - else { - var current = start - var prev: WeakConcurrentNode[N] = null + var current = start + var name: N = null - var currentName: N = null - var found: N = null + do { + name = current.name + if ((name eq null) || name.id.hashCode() != hash || name.id != key) { + name = null + current = current.next + } + } while ((name eq null) && (current ne null) && (current ne end)) + name + } - do { - currentName = current.name - if ((currentName ne null) && currentName.id.hashCode() == hash && currentName.id == key) - found = currentName - else { - prev = current - current = current.next - } - } while ((found eq null) && (current ne null) && (current ne end)) - found - } + final def findNoTrim[N >: Null <: NameBase](start: WeakConcurrentNode[N], key: String, hash: Int): N = { + var current = start + var name: N = null + + do { + name = current.name + if ((name eq null) || name.id.hashCode() != hash || name.id != key) { + name = null + current = current.next + } + } while ((name eq null) && (current ne null)) + name } /** trim the collected nodes @@ -217,7 +214,7 @@ object WeakConcurrentNode { * * @return the count of collected nodes */ - def trimCollected[N >: Null <: NameBase](start: WeakConcurrentNode[N]) : Int= { + def trimCollected[N >: Null <: NameBase](start: WeakConcurrentNode[N]): Int = { var trimmed = 0 var prev: WeakConcurrentNode[N] = start while (prev ne null) { @@ -234,43 +231,44 @@ object WeakConcurrentNode { trimmed } -// final def findAutoTrim[N >: Null <: Name](start: WeakConcurrentNode[N], key: String, hash: Int, end: Node[N], decrementOnRemove: AtomicInteger): N = { -// if (start eq end) -// null -// else { -// //prev refers to the previos entry with a name that has not been GCed -// var prevRef: WeakConcurrentNode[N] = null -// var prevRefName: N = null -// -// var current = start -// var currentName: N = null -// var found: N = null -// -// var skipped = 0 -// -// do { -// currentName = current.name -// while ((current ne null) && (currentName eq null)) { -// val next = current.next -// skipped += 1 -// } -// } -// //we dont care if we lost the race - someone else fixed it -// if ((prev ne null) && casNext(prev, current, next)) -// decrementOnRemove.decrementAndGet() -// //leave prev where it was -// current = next -// } else if (currentName.id.hashCode() == hash && currentName.id == key) -// found = currentName -// else { -// prev = current -// current = current.next -// } -// } while ((found eq null) && (current ne null) && (current ne end)) -// found -// } -// } + // final def findAutoTrim[N >: Null <: Name](start: WeakConcurrentNode[N], key: String, hash: Int, end: Node[N], decrementOnRemove: AtomicInteger): N = { + // if (start eq end) + // null + // else { + // //prev refers to the previos entry with a name that has not been GCed + // var prevRef: WeakConcurrentNode[N] = null + // var prevRefName: N = null + // + // var current = start + // var currentName: N = null + // var found: N = null + // + // var skipped = 0 + // + // do { + // currentName = current.name + // while ((current ne null) && (currentName eq null)) { + // val next = current.next + // skipped += 1 + // } + // } + // //we dont care if we lost the race - someone else fixed it + // if ((prev ne null) && casNext(prev, current, next)) + // decrementOnRemove.decrementAndGet() + // //leave prev where it was + // current = next + // } else if (currentName.id.hashCode() == hash && currentName.id == key) + // found = currentName + // else { + // prev = current + // current = current.next + // } + // } while ((found eq null) && (current ne null) && (current ne end)) + // found + // } + // } } + final class WeakConcurrentNode[N >: Null <: NameBase](private val ref: WeakReference[N], @volatile var next: WeakConcurrentNode[N]) extends Node[N] { private def nextAccess = AtomicReferenceFieldUpdater.newUpdater(classOf[WeakConcurrentNode[_]], classOf[WeakConcurrentNode[_]], "next") @@ -282,6 +280,7 @@ final class WeakConcurrentNode[N >: Null <: NameBase](private val ref: WeakRefer } def name = ref.get() + def isDefined = ref.get() ne null def hash: Int = { diff --git a/src/reflect/scala/reflect/internal/names/StrongConcurrentNodeInterner.scala b/src/reflect/scala/reflect/internal/names/StrongConcurrentNodeInterner.scala index e070a28edc0..2bd7a8188e9 100644 --- a/src/reflect/scala/reflect/internal/names/StrongConcurrentNodeInterner.scala +++ b/src/reflect/scala/reflect/internal/names/StrongConcurrentNodeInterner.scala @@ -41,14 +41,16 @@ class StrongConcurrentNodeInterner[T >: Null <: NameBase](createName: String => if (head ne null) name = StrongNode.find(head, key, hash, oldTail) if (name eq null) { - // minor optimisation - we can skip this tail if we have to retry - // if we came to the end of the chain of nodes we dont need to search the same tail if we fail and try again - oldTail = head name = createName(key) val newNode = new N(name, head) if (data.compareAndSet(bucket, head, newNode)) { afterInsert(data) - } else name = null + } else { + // minor optimisation - we can skip this tail if we have to retry + // if we came to the end of the chain of nodes we dont need to search the same tail if we fail and try again + oldTail = head + name = null + } } } } while (name eq null) diff --git a/src/reflect/scala/reflect/internal/names/WeakFixedSizeConcurrentNodeInterner.scala b/src/reflect/scala/reflect/internal/names/WeakFixedSizeConcurrentNodeInterner.scala index 022f0f07402..70e101ea4bd 100644 --- a/src/reflect/scala/reflect/internal/names/WeakFixedSizeConcurrentNodeInterner.scala +++ b/src/reflect/scala/reflect/internal/names/WeakFixedSizeConcurrentNodeInterner.scala @@ -33,9 +33,31 @@ sealed abstract class AbstractWeakFixedSizeConcurrentNodeInterner[T >: Null <: N if (head ne null) name = WeakConcurrentNode.findNoTrim(head, key, hash, oldTail) if (name eq null) { - // minor optimisation - we can skip this tail if we have to retry - // if we came to the end of the chain of nodes we dont need to search the same tail if we fail and try again - oldTail = head + name = createName(key) + val newNode = createNode(name, head, bucket) + if (data.compareAndSet(bucket, head, newNode)) { + size_.incrementAndGet() + } else { + name = null + // minor optimisation - we can skip this tail if we have to retry + // if we came to the end of the chain of nodes we dont need to search the same tail if we fail and try again + oldTail = head + } + } + } while (name eq null) + name + } + def findNoTail(key: String): T = { + val hash = key.hashCode + val improved = improveHash(hash) + + var name: T = null + val bucket = improved & mask + do { + val head = data.get(bucket) + if (head ne null) + name = WeakConcurrentNode.findNoTrim(head, key, hash) + if (name eq null) { name = createName(key) val newNode = createNode(name, head, bucket) if (data.compareAndSet(bucket, head, newNode)) { @@ -50,17 +72,13 @@ sealed abstract class AbstractWeakFixedSizeConcurrentNodeInterner[T >: Null <: N def find1(chars: Array[Char], start: Int, count: Int): T = { val hash = hashLikeString(chars, start, count) val improved = improveHash(hash) - var oldTail: N = null var name: T = null val bucket = improved & mask val head = data.get(bucket) if (head ne null) - name = WeakConcurrentNode.findNoTrimC1(head, chars, start, count, hash, oldTail) + name = WeakConcurrentNode.findNoTrimC1(head, chars, start, count, hash) if (name eq null) { - // minor optimisation - we can skip this tail if we have to retry - // if we came to the end of the chain of nodes we dont need to search the same tail if we fail and try again - oldTail = head val string = new String(chars, start, count) name = createName(string) val newNode = createNode(name, head, bucket) @@ -75,17 +93,13 @@ sealed abstract class AbstractWeakFixedSizeConcurrentNodeInterner[T >: Null <: N def find2(chars: Array[Char], start: Int, count: Int): T = { val hash = hashLikeString(chars, start, count) val improved = improveHash(hash) - var oldTail: N = null var name: T = null val bucket = improved & mask val head = data.get(bucket) if (head ne null) - name = WeakConcurrentNode.findNoTrimC2(head, chars, start, count, hash, oldTail) + name = WeakConcurrentNode.findNoTrimC2(head, chars, start, count, hash) if (name eq null) { - // minor optimisation - we can skip this tail if we have to retry - // if we came to the end of the chain of nodes we dont need to search the same tail if we fail and try again - oldTail = head val string = new String(chars, start, count) name = createName(string) val newNode = createNode(name, head, bucket) @@ -100,17 +114,13 @@ sealed abstract class AbstractWeakFixedSizeConcurrentNodeInterner[T >: Null <: N def find3(chars: Array[Char], start: Int, count: Int): T = { val hash = hashLikeString(chars, start, count) val improved = improveHash(hash) - var oldTail: N = null var name: T = null val bucket = improved & mask val head = data.get(bucket) if (head ne null) - name = WeakConcurrentNode.findNoTrimC3(head, chars, start, count, hash, oldTail) + name = WeakConcurrentNode.findNoTrimC3(head, chars, start, count, hash) if (name eq null) { - // minor optimisation - we can skip this tail if we have to retry - // if we came to the end of the chain of nodes we dont need to search the same tail if we fail and try again - oldTail = head val string = new String(chars, start, count) name = createName(string) val newNode = createNode(name, head, bucket) @@ -130,7 +140,8 @@ sealed abstract class AbstractWeakFixedSizeConcurrentNodeInterner[T >: Null <: N val improved = improveHash(hash) val head = data.get(improved & (data.length - 1)) - WeakConcurrentNode.findNoTrim(head, key, hash, null) + if (head eq null) null + else WeakConcurrentNode.findNoTrim(head, key, hash) } @@ -221,6 +232,7 @@ object AbstractUnsafeWeakFixedSizeConcurrentNodeInterner { throw new Error("data type scale not a power of two") 31 - Integer.numberOfLeadingZeros(scale) } + @inline private[this] def offset(bucket: Int) = (bucket.toLong << shift) + base @@ -253,9 +265,31 @@ sealed abstract class AbstractUnsafeWeakFixedSizeConcurrentNodeInterner[T >: Nul if (head ne null) name = WeakConcurrentNode.findNoTrim(head, key, hash, oldTail) if (name eq null) { - // minor optimisation - we can skip this tail if we have to retry - // if we came to the end of the chain of nodes we dont need to search the same tail if we fail and try again - oldTail = head + name = createName(key) + val newNode = createNode(name, head, bucket) + if (compareAndSet(data, bucket, head, newNode)) { + size_.incrementAndGet() + } else { + name = null + // minor optimisation - we can skip this tail if we have to retry + // if we came to the end of the chain of nodes we dont need to search the same tail if we fail and try again + oldTail = head + } + } + } while (name eq null) + name + } + def findNoTail(key: String): T = { + val hash = key.hashCode + val improved = improveHash(hash) + + var name: T = null + val bucket = improved & mask + do { + val head = getVolatile(data, bucket) + if (head ne null) + name = WeakConcurrentNode.findNoTrim(head, key, hash) + if (name eq null) { name = createName(key) val newNode = createNode(name, head, bucket) if (compareAndSet(data, bucket, head, newNode)) { @@ -275,7 +309,8 @@ sealed abstract class AbstractUnsafeWeakFixedSizeConcurrentNodeInterner[T >: Nul val improved = improveHash(hash) val head = getVolatile(data, improved & (data.length - 1)) - WeakConcurrentNode.findNoTrim(head, key, hash, null) + if (head eq null) null + else WeakConcurrentNode.findNoTrim(head, key, hash) } @@ -300,6 +335,7 @@ sealed abstract class AbstractUnsafeWeakFixedSizeConcurrentNodeInterner[T >: Nul @inline final private def mask = (1 << 16) - 1 } + class UnsafeWeakFixedSizeNoAutoTrimConcurrentNodeInterner[T >: Null <: NameBase](createName: String => T) extends AbstractUnsafeWeakFixedSizeConcurrentNodeInterner(createName) { override def createNode(name: T, next: WeakConcurrentNode[T], bucket: Int) = new WeakConcurrentNode[T](new WeakReference(name), next) @@ -308,6 +344,7 @@ class UnsafeWeakFixedSizeNoAutoTrimConcurrentNodeInterner[T >: Null <: NameBase] trimImpl(index) } } + class UnsafeWeakFixedSizeAutoTrimConcurrentNodeInterner[T >: Null <: NameBase](createName: String => T) extends AbstractUnsafeWeakFixedSizeConcurrentNodeInterner(createName) { private val queue = new ReferenceQueue[T] diff --git a/test/benchmarks/src/main/scala/scala/reflect/NamesBenchmark.scala b/test/benchmarks/src/main/scala/scala/reflect/NamesBenchmark.scala index 950794c74f6..889cf63941e 100644 --- a/test/benchmarks/src/main/scala/scala/reflect/NamesBenchmark.scala +++ b/test/benchmarks/src/main/scala/scala/reflect/NamesBenchmark.scala @@ -55,6 +55,12 @@ abstract class AbstractNamesBenchmark { case "Unsafe" => () => new UnsafeWeakFixedSizeNoAutoTrimConcurrentNodeInterner(new NameBase(_)) { override def find(chars: Array[Char], start: Int, count: Int) = super.find(chars, start, count) } + case "Tail" => () => new WeakFixedSizeNoAutoTrimConcurrentNodeInterner(new NameBase(_)) { + override def find(key: String) = super.find(key) + } + case "NoTail" => () => new UnsafeWeakFixedSizeNoAutoTrimConcurrentNodeInterner(new NameBase(_)) { + override def find(key: String) = super.findNoTail(key) + } } fullNameTable = newNameTable() names foreach {fullNameTable.find(_)} @@ -82,7 +88,7 @@ abstract class AbstractNamesBenchmark { class NamesBenchmark extends AbstractNamesBenchmark { @Param(Array("Existing", "ExistingSynchronized", "MapNameTable", "ConcurrentMapNameTable", "WeakConcurrentMapNameTable", "StrongConcurrentNodeInterner", "WeakFixedSizeNoAutoTrimConcurrentNodeInterner", - "WeakFixedSizeAutoTrimConcurrentNodeInterner", "Find0", "Find1", "Find2", "Find3", "Safe", "Unsafe")) + "WeakFixedSizeAutoTrimConcurrentNodeInterner", "Find0", "Find1", "Find2", "Find3", "Safe", "Unsafe", "Tail", "NoTail")) var testImpl: String = _ override def testType = testImpl @Benchmark def findNewString(bh: Blackhole): Any = { @@ -108,7 +114,7 @@ class NamesBenchmark extends AbstractNamesBenchmark { class NamesBenchmarkMT extends AbstractNamesBenchmark { @Param(Array("ExistingSynchronized", "ConcurrentMapNameTable", "WeakConcurrentMapNameTable", "StrongConcurrentNodeInterner", "WeakFixedSizeNoAutoTrimConcurrentNodeInterner", - "WeakFixedSizeAutoTrimConcurrentNodeInterner", "Find0", "Find1", "Find2", "Find3", "Safe", "Unsafe")) + "WeakFixedSizeAutoTrimConcurrentNodeInterner", "Find0", "Find1", "Find2", "Find3", "Safe", "Unsafe", "Tail", "NoTail")) var testImpl: String = _ override def testType = testImpl } \ No newline at end of file diff --git a/test/junit/scala/reflect/internal/names/NamesImplTest.scala b/test/junit/scala/reflect/internal/names/NamesImplTest.scala index ef0a47c2427..d2f4d810ab8 100644 --- a/test/junit/scala/reflect/internal/names/NamesImplTest.scala +++ b/test/junit/scala/reflect/internal/names/NamesImplTest.scala @@ -65,3 +65,15 @@ class Unsafe extends ExtendedNameTest { override type T = Term override val nameTable: NameTable[T] = new UnsafeWeakFixedSizeAutoTrimConcurrentNodeInterner[Term](Term.apply) } +class Tail extends ExtendedNameTest { + override type T = Term + override val nameTable: NameTable[T] = new WeakFixedSizeAutoTrimConcurrentNodeInterner[Term](Term.apply) { + override def find(key: String): Term = super.find(key) + } +} +class NoTail extends ExtendedNameTest { + override type T = Term + override val nameTable: NameTable[T] = new WeakFixedSizeAutoTrimConcurrentNodeInterner[Term](Term.apply) { + override def find(key: String): Term = super.findNoTail(key) + } +} From 7a2436e13b78015261bfc2001e094f22fe97bd8b Mon Sep 17 00:00:00 2001 From: Mike Skells Date: Sat, 22 Jun 2019 07:47:04 +0100 Subject: [PATCH 5/7] benchmark improvements - no GC while building the table --- .../scala/scala/reflect/NamesBenchmark.scala | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/test/benchmarks/src/main/scala/scala/reflect/NamesBenchmark.scala b/test/benchmarks/src/main/scala/scala/reflect/NamesBenchmark.scala index 889cf63941e..e89f16dc878 100644 --- a/test/benchmarks/src/main/scala/scala/reflect/NamesBenchmark.scala +++ b/test/benchmarks/src/main/scala/scala/reflect/NamesBenchmark.scala @@ -22,8 +22,8 @@ abstract class AbstractNamesBenchmark { def testType: String - var fullNameTable: NameTable[_] = _ - var newNameTable: () => NameTable[_] = _ + var fullNameTable: NameTable[_ <: AnyRef] = _ + var newNameTable: () => NameTable[_ <: AnyRef] = _ @Setup def setup(): Unit = { names = Array.tabulate(arraySize)(i => s"name$i") @@ -67,21 +67,25 @@ abstract class AbstractNamesBenchmark { } @Benchmark def findOldString(bh: Blackhole): Any = { - val nameTable: NameTable[_] = fullNameTable + val nameTable: NameTable[_ <: AnyRef] = fullNameTable + val res = new Array[AnyRef](arraySize) var i = 0 while (i < arraySize) { - bh.consume(nameTable.find(names(i))) + res(i) = nameTable.find(names(i)) i += 1 } + bh.consume(res) } @Benchmark def findOldChars(bh: Blackhole): Any = { - val nameTable: NameTable[_] = fullNameTable + val nameTable: NameTable[_<: AnyRef] = fullNameTable var i = 0 + val res = new Array[AnyRef](arraySize) while (i < arraySize) { val c= namesChars(i) - bh.consume(nameTable.find(c,0,c.length)) + res(i) = nameTable.find(c,0,c.length) i += 1 } + bh.consume(res) } } @Threads(1) @@ -93,21 +97,25 @@ class NamesBenchmark extends AbstractNamesBenchmark { override def testType = testImpl @Benchmark def findNewString(bh: Blackhole): Any = { val nameTable = newNameTable() + val res = new Array[AnyRef](arraySize) var i = 0 while (i < arraySize) { - bh.consume(nameTable.find(names(i))) + res(i) = nameTable.find(names(i)) i += 1 } + bh.consume(res) } @Benchmark def findNewChars(bh: Blackhole): Any = { val nameTable = newNameTable() var i = 0 + val res = new Array[AnyRef](arraySize) while (i < arraySize) { val c= namesChars(i) - bh.consume(nameTable.find(c,0,c.length)) + res(i) = nameTable.find(c,0,c.length) i += 1 } + bh.consume(res) } } @Threads(4) From 2b2605470e8f11483896cb830c41c7bcfe8f2f13 Mon Sep 17 00:00:00 2001 From: Mike Skells Date: Mon, 24 Jun 2019 21:14:04 +0100 Subject: [PATCH 6/7] more optimisations add test assertions on allocations. --- .../nsc/profile/ExtendedThreadMxBean.java | 1 + .../names/ConcurrentMapNameTable.scala | 32 +++- .../reflect/internal/names/MapNameTable.scala | 2 + .../reflect/internal/names/NameTable.scala | 5 + .../names/WeakConcurrentMapNameTable.scala | 149 ++++++++++++++---- .../scala/reflect/ExistingNameTable.scala | 2 + .../scala/scala/reflect/NamesBenchmark.scala | 101 +++++++----- .../internal/names/ExtendedNameTest.scala | 87 ++++++++-- .../internal/names/NamesImplTest.scala | 32 +++- test/junit/scala/util/Allocations.scala | 36 +++++ 10 files changed, 360 insertions(+), 87 deletions(-) create mode 100644 test/junit/scala/util/Allocations.scala diff --git a/src/compiler/scala/tools/nsc/profile/ExtendedThreadMxBean.java b/src/compiler/scala/tools/nsc/profile/ExtendedThreadMxBean.java index 1d5cf4bc3e4..7db8bbe37b4 100644 --- a/src/compiler/scala/tools/nsc/profile/ExtendedThreadMxBean.java +++ b/src/compiler/scala/tools/nsc/profile/ExtendedThreadMxBean.java @@ -21,6 +21,7 @@ @SuppressWarnings("unused") public abstract class ExtendedThreadMxBean implements ThreadMXBean { static final ExtendedThreadMxBean proxy; + public static ExtendedThreadMxBean proxy() {return proxy;} static { ExtendedThreadMxBean local; diff --git a/src/reflect/scala/reflect/internal/names/ConcurrentMapNameTable.scala b/src/reflect/scala/reflect/internal/names/ConcurrentMapNameTable.scala index 6aa3d2ae5c3..8fe42bf2562 100644 --- a/src/reflect/scala/reflect/internal/names/ConcurrentMapNameTable.scala +++ b/src/reflect/scala/reflect/internal/names/ConcurrentMapNameTable.scala @@ -2,11 +2,41 @@ package scala.reflect.internal.names import java.util.concurrent.ConcurrentHashMap -class ConcurrentMapNameTable[T <: AnyRef](builder: (String) => T) extends NameTable[T] { +class ConcurrentMapNameTable1[T <: AnyRef](builder: (String) => T) extends NameTable[T] { override def size: Int = data.size private val data = new ConcurrentHashMap[String, T] def find(source: String): T = data.computeIfAbsent(source, (s) => builder(s)) + + override def nonAllocatingStringLookup: Boolean = false +} +class ConcurrentMapNameTable2[T <: AnyRef](builder: (String) => T) extends NameTable[T] { + + override def size: Int = data.size + + private val data = new ConcurrentHashMap[String, T] + private val builderFn = new java.util.function.Function[String, T]{ + override def apply(t: String): T = builder(t) + } + + def find(source: String): T = data.computeIfAbsent(source, builderFn) +} +class ConcurrentMapNameTable3[T <: AnyRef](builder: (String) => T) extends NameTable[T] { + + override def size: Int = data.size + + private val data = new ConcurrentHashMap[String, T] + + def find(source: String): T = { + var res = data.get(source) + if (res eq null) { + res = builder(source) + val existing = data.putIfAbsent(source, res) + if (existing ne null) + res = existing + } + res + } } diff --git a/src/reflect/scala/reflect/internal/names/MapNameTable.scala b/src/reflect/scala/reflect/internal/names/MapNameTable.scala index 086878d5de8..4b9d56ecccd 100644 --- a/src/reflect/scala/reflect/internal/names/MapNameTable.scala +++ b/src/reflect/scala/reflect/internal/names/MapNameTable.scala @@ -7,4 +7,6 @@ class MapNameTable[T <: AnyRef](builder: (String) => T) extends NameTable[T] { private val data = new mutable.AnyRefMap[String, T] def find(source: String): T = data.getOrElseUpdate(source, builder(source)) + + override def nonAllocatingStringLookup: Boolean = false } diff --git a/src/reflect/scala/reflect/internal/names/NameTable.scala b/src/reflect/scala/reflect/internal/names/NameTable.scala index eb07427befe..da4a305a326 100644 --- a/src/reflect/scala/reflect/internal/names/NameTable.scala +++ b/src/reflect/scala/reflect/internal/names/NameTable.scala @@ -3,6 +3,11 @@ package scala.reflect.internal.names import java.nio.charset.Charset abstract class NameTable[T <: AnyRef] { + //temp method + def nonAllocatingCharLookup: Boolean = false + //temp method + def nonAllocatingStringLookup: Boolean = true + def size: Int def find(source: String): T diff --git a/src/reflect/scala/reflect/internal/names/WeakConcurrentMapNameTable.scala b/src/reflect/scala/reflect/internal/names/WeakConcurrentMapNameTable.scala index f6e8369054a..81cac271192 100644 --- a/src/reflect/scala/reflect/internal/names/WeakConcurrentMapNameTable.scala +++ b/src/reflect/scala/reflect/internal/names/WeakConcurrentMapNameTable.scala @@ -1,18 +1,100 @@ package scala.reflect.internal.names import java.lang.ref.{ReferenceQueue, WeakReference} +import java.util.Map import java.util.concurrent.ConcurrentHashMap +import java.util.function.Consumer import scala.annotation.tailrec -final class WeakConcurrentMapNameTable[T >: Null <: AnyRef](builder: (String) => T) extends NameTable[T] { +abstract class AbstractWeakConcurrentMapNameTable1[T >: Null <: AnyRef](builder: (String) => T) extends NameTable[T] { - class NameRef(val key: String, name: T) extends WeakReference(name, queue) + type REF <: WeakReference[T] + def newNameRef(key:String, name: T):REF + + protected val data = new ConcurrentHashMap[String, REF] + + override def size: Int = data.size + + def find(source: String): T = { + var res: T = null + //we assume that we usually find a match, so no lock on the fast path + var entry = data.get(source) + + var maybe: T = null + do { + if (entry eq null) { + if (maybe eq null) maybe = builder(source) + entry = data.putIfAbsent(source, newNameRef(source, maybe)) + if (entry eq null) + res = maybe + } else { + res = entry.get() + if (res eq null) { + if (maybe eq null) maybe = builder(source) + if (data.replace(source, entry, newNameRef(source, maybe))) + res = maybe + else { + res = null + entry = data.get(source) + } + } + } + } while (res eq null) + res + } +} +abstract class AbstractWeakConcurrentMapNameTable2[T >: Null <: AnyRef](builder: (String) => T) extends NameTable[T] { + + type REF <: WeakReference[T] + def newNameRef(key:String, name: T):REF + + protected val data = new ConcurrentHashMap[String, REF] - private def newNameRef(s: String) = new NameRef(s, builder(s)) + override def size: Int = data.size + + private val remapper = new java.util.function.BiFunction[String, REF, REF] { + override def apply(key: String, existing: REF): REF = + if ((existing eq null) || (existing.get eq null)) + newNameRef(key, builder(key)) + else existing + } + def find(source: String): T = { + var res: T = null + do res = data.compute(source, remapper).get + while ( res eq null) + res + } +} +final class WeakNoAutoTrimConcurrentMapNameTable1[T >: Null <: AnyRef](builder: (String) => T) extends AbstractWeakConcurrentMapNameTable1(builder) { + type REF = WeakReference[T] + override def newNameRef(key:String, name: T):REF = new WeakReference(name) + + private val removeCollected = new Consumer[Map.Entry[String, REF]] { + override def accept(t: Map.Entry[String, REF]): Unit = + if (t.getValue.get() == null) data.remove(t.getKey, t.getValue) + } + + def trim() = data.forEachEntry(1, removeCollected) +} +final class WeakNoAutoTrimConcurrentMapNameTable2[T >: Null <: AnyRef](builder: (String) => T) extends AbstractWeakConcurrentMapNameTable2(builder) { + type REF = WeakReference[T] + override def newNameRef(key:String, name: T):REF = new WeakReference(name) + + private val removeCollected = new Consumer[Map.Entry[String, REF]] { + override def accept(t: Map.Entry[String, REF]): Unit = + if (t.getValue.get() == null) data.remove(t.getKey, t.getValue) + } + + def trim() = data.forEachEntry(1, removeCollected) +} +final class WeakAutoTrimConcurrentMapNameTable1[T >: Null <: AnyRef](builder: (String) => T) extends AbstractWeakConcurrentMapNameTable1(builder) { - private val data = new ConcurrentHashMap[String, NameRef] private val queue = new ReferenceQueue[T] + type REF = NameRef + class NameRef(val key: String, name: T) extends WeakReference(name, queue) + + override def newNameRef(key: String, name: T) = new NameRef(key, name) @tailrec def processQueue(): Unit = { val next = queue.poll().asInstanceOf[NameRef] @@ -31,33 +113,44 @@ final class WeakConcurrentMapNameTable[T >: Null <: AnyRef](builder: (String) => override def size: Int = { processQueue() - data.size + super.size } - def find(source: String): T = { + override def find(source: String): T = { processQueue() - var res: T = null - //we assume that we usually find a match, so no lock on the fast path - var entry = data.get(source) - do { - if (entry eq null) { - val maybe = builder(source) - entry = data.putIfAbsent(source, new NameRef(source, maybe)) - if (entry eq null) - res = maybe - } else { - res = entry.get() - if (res eq null) { - val maybe = builder(source) - if (data.replace(source, entry, new NameRef(source, maybe))) - res = maybe - else { - res = null - entry = data.get(source) - } - } + super.find(source) + } +} +final class WeakAutoTrimConcurrentMapNameTable2[T >: Null <: AnyRef](builder: (String) => T) extends AbstractWeakConcurrentMapNameTable2(builder) { + + private val queue = new ReferenceQueue[T] + type REF = NameRef + class NameRef(val key: String, name: T) extends WeakReference(name, queue) + + override def newNameRef(key: String, name: T) = new NameRef(key, name) + + @tailrec def processQueue(): Unit = { + val next = queue.poll().asInstanceOf[NameRef] + if (next ne null) { + data.get(next.key) match { + case null => + case ref => + if (ref.get() eq null) + data.compute(ref.key, (_: String, existing: NameRef) => + if (existing.get eq null) null + else existing) } - } while (res eq null) - res + processQueue() + } + } + + override def size: Int = { + processQueue() + super.size + } + + override def find(source: String): T = { + processQueue() + super.find(source) } } diff --git a/test/benchmarks/src/main/scala/scala/reflect/ExistingNameTable.scala b/test/benchmarks/src/main/scala/scala/reflect/ExistingNameTable.scala index bf2a61d7998..be8d17e3994 100644 --- a/test/benchmarks/src/main/scala/scala/reflect/ExistingNameTable.scala +++ b/test/benchmarks/src/main/scala/scala/reflect/ExistingNameTable.scala @@ -9,6 +9,8 @@ class ExistingSynchronizedNameTable extends ExistingNameTable{ class ExistingNameTable extends NameTable[AnyRef] with Names{ override def size: Int = -1 + override def nonAllocatingCharLookup: Boolean = true + override def find(source: String) = newTermName(source.toCharArray(), 0, source.length(), null) diff --git a/test/benchmarks/src/main/scala/scala/reflect/NamesBenchmark.scala b/test/benchmarks/src/main/scala/scala/reflect/NamesBenchmark.scala index e89f16dc878..b79111bc4f8 100644 --- a/test/benchmarks/src/main/scala/scala/reflect/NamesBenchmark.scala +++ b/test/benchmarks/src/main/scala/scala/reflect/NamesBenchmark.scala @@ -17,7 +17,7 @@ abstract class AbstractNamesBenchmark { var names: Array[String] = null var namesChars: Array[Array[Char]] = null - @Param(Array("0", "10", "100","1000", "10000", "100000")) + @Param(Array("0", "10", "10000", "100000", "1000000")) var arraySize: Int = _ def testType: String @@ -31,39 +31,54 @@ abstract class AbstractNamesBenchmark { newNameTable = testType match { case "Existing" => () => new ExistingNameTable case "ExistingSynchronized" => () => new ExistingSynchronizedNameTable - case "MapNameTable" => () => new MapNameTable (new NameBase(_)) - case "ConcurrentMapNameTable" => () => new ConcurrentMapNameTable(new NameBase(_)) - case "WeakConcurrentMapNameTable" => () => new WeakConcurrentMapNameTable(new NameBase(_)) + case "MapNameTable" => () => new MapNameTable(new NameBase(_)) + case "ConcurrentMapNameTable1" => () => new ConcurrentMapNameTable1(new NameBase(_)) + case "ConcurrentMapNameTable2" => () => new ConcurrentMapNameTable2(new NameBase(_)) + case "ConcurrentMapNameTable3" => () => new ConcurrentMapNameTable3(new NameBase(_)) + case "WeakAutoTrimConcurrentMapNameTable1" => () => new WeakAutoTrimConcurrentMapNameTable1[AnyRef](new NameBase(_)) + case "WeakAutoTrimConcurrentMapNameTable2" => () => new WeakAutoTrimConcurrentMapNameTable1[AnyRef](new NameBase(_)) + case "WeakNoAutoTrimConcurrentMapNameTable1" => () => new WeakNoAutoTrimConcurrentMapNameTable1[AnyRef](new NameBase(_)) + case "WeakNoAutoTrimConcurrentMapNameTable2" => () => new WeakNoAutoTrimConcurrentMapNameTable2[AnyRef](new NameBase(_)) case "StrongConcurrentNodeInterner" => () => new StrongConcurrentNodeInterner(new NameBase(_)) case "WeakFixedSizeNoAutoTrimConcurrentNodeInterner" => () => new WeakFixedSizeNoAutoTrimConcurrentNodeInterner(new NameBase(_)) case "WeakFixedSizeAutoTrimConcurrentNodeInterner" => () => new WeakFixedSizeAutoTrimConcurrentNodeInterner(new NameBase(_)) - case "Find0" => () => new WeakFixedSizeNoAutoTrimConcurrentNodeInterner(new NameBase(_)) { - override def find(chars: Array[Char], start: Int, count: Int) = super.find(chars, start, count) - } - case "Find1" => () => new WeakFixedSizeNoAutoTrimConcurrentNodeInterner(new NameBase(_)) { - override def find(chars: Array[Char], start: Int, count: Int) = super.find1(chars, start, count) - } - case "Find2" => () => new WeakFixedSizeNoAutoTrimConcurrentNodeInterner(new NameBase(_)) { - override def find(chars: Array[Char], start: Int, count: Int) = super.find2(chars, start, count) - } - case "Find3" => () => new WeakFixedSizeNoAutoTrimConcurrentNodeInterner(new NameBase(_)) { - override def find(chars: Array[Char], start: Int, count: Int) = super.find3(chars, start, count) - } - case "Safe" => () => new WeakFixedSizeNoAutoTrimConcurrentNodeInterner(new NameBase(_)) { - override def find(chars: Array[Char], start: Int, count: Int) = super.find(chars, start, count) - } - case "Unsafe" => () => new UnsafeWeakFixedSizeNoAutoTrimConcurrentNodeInterner(new NameBase(_)) { - override def find(chars: Array[Char], start: Int, count: Int) = super.find(chars, start, count) - } - case "Tail" => () => new WeakFixedSizeNoAutoTrimConcurrentNodeInterner(new NameBase(_)) { - override def find(key: String) = super.find(key) - } - case "NoTail" => () => new UnsafeWeakFixedSizeNoAutoTrimConcurrentNodeInterner(new NameBase(_)) { - override def find(key: String) = super.findNoTail(key) - } + case "Find0" => () => + new WeakFixedSizeNoAutoTrimConcurrentNodeInterner(new NameBase(_)) { + override def find(chars: Array[Char], start: Int, count: Int) = super.find(chars, start, count) + } + case "Find1" => () => + new WeakFixedSizeNoAutoTrimConcurrentNodeInterner(new NameBase(_)) { + override def find(chars: Array[Char], start: Int, count: Int) = super.find1(chars, start, count) + } + case "Find2" => () => + new WeakFixedSizeNoAutoTrimConcurrentNodeInterner(new NameBase(_)) { + override def find(chars: Array[Char], start: Int, count: Int) = super.find2(chars, start, count) + } + case "Find3" => () => + new WeakFixedSizeNoAutoTrimConcurrentNodeInterner(new NameBase(_)) { + override def find(chars: Array[Char], start: Int, count: Int) = super.find3(chars, start, count) + } + case "Safe" => () => + new WeakFixedSizeNoAutoTrimConcurrentNodeInterner(new NameBase(_)) { + override def find(chars: Array[Char], start: Int, count: Int) = super.find(chars, start, count) + } + case "Unsafe" => () => + new UnsafeWeakFixedSizeNoAutoTrimConcurrentNodeInterner(new NameBase(_)) { + override def find(chars: Array[Char], start: Int, count: Int) = super.find(chars, start, count) + } + case "Tail" => () => + new WeakFixedSizeNoAutoTrimConcurrentNodeInterner(new NameBase(_)) { + override def find(key: String) = super.find(key) + } + case "NoTail" => () => + new UnsafeWeakFixedSizeNoAutoTrimConcurrentNodeInterner(new NameBase(_)) { + override def find(key: String) = super.findNoTail(key) + } } fullNameTable = newNameTable() - names foreach {fullNameTable.find(_)} + names foreach { + fullNameTable.find(_) + } } @Benchmark def findOldString(bh: Blackhole): Any = { @@ -76,25 +91,32 @@ abstract class AbstractNamesBenchmark { } bh.consume(res) } + @Benchmark def findOldChars(bh: Blackhole): Any = { - val nameTable: NameTable[_<: AnyRef] = fullNameTable + val nameTable: NameTable[_ <: AnyRef] = fullNameTable var i = 0 val res = new Array[AnyRef](arraySize) while (i < arraySize) { - val c= namesChars(i) - res(i) = nameTable.find(c,0,c.length) + val c = namesChars(i) + res(i) = nameTable.find(c, 0, c.length) i += 1 } bh.consume(res) } } + @Threads(1) class NamesBenchmark extends AbstractNamesBenchmark { - @Param(Array("Existing", "ExistingSynchronized", "MapNameTable", "ConcurrentMapNameTable", - "WeakConcurrentMapNameTable", "StrongConcurrentNodeInterner", "WeakFixedSizeNoAutoTrimConcurrentNodeInterner", + @Param(Array("Existing", "ExistingSynchronized", "MapNameTable", + "ConcurrentMapNameTable1", "ConcurrentMapNameTable2", "ConcurrentMapNameTable3", + "WeakAutoTrimConcurrentMapNameTable1", "WeakAutoTrimConcurrentMapNameTable2", + "WeakNoAutoTrimConcurrentMapNameTable1", "WeakNoAutoTrimConcurrentMapNameTable2", + "StrongConcurrentNodeInterner", "WeakFixedSizeNoAutoTrimConcurrentNodeInterner", "WeakFixedSizeAutoTrimConcurrentNodeInterner", "Find0", "Find1", "Find2", "Find3", "Safe", "Unsafe", "Tail", "NoTail")) var testImpl: String = _ + override def testType = testImpl + @Benchmark def findNewString(bh: Blackhole): Any = { val nameTable = newNameTable() val res = new Array[AnyRef](arraySize) @@ -111,18 +133,23 @@ class NamesBenchmark extends AbstractNamesBenchmark { var i = 0 val res = new Array[AnyRef](arraySize) while (i < arraySize) { - val c= namesChars(i) - res(i) = nameTable.find(c,0,c.length) + val c = namesChars(i) + res(i) = nameTable.find(c, 0, c.length) i += 1 } bh.consume(res) } } + @Threads(4) class NamesBenchmarkMT extends AbstractNamesBenchmark { - @Param(Array("ExistingSynchronized", "ConcurrentMapNameTable", "WeakConcurrentMapNameTable", + @Param(Array("ExistingSynchronized", + "ConcurrentMapNameTable1", "ConcurrentMapNameTable2", "ConcurrentMapNameTable3", + "WeakAutoTrimConcurrentMapNameTable1", "WeakAutoTrimConcurrentMapNameTable2", + "WeakNoAutoTrimConcurrentMapNameTable1", "WeakNoAutoTrimConcurrentMapNameTable2", "StrongConcurrentNodeInterner", "WeakFixedSizeNoAutoTrimConcurrentNodeInterner", "WeakFixedSizeAutoTrimConcurrentNodeInterner", "Find0", "Find1", "Find2", "Find3", "Safe", "Unsafe", "Tail", "NoTail")) var testImpl: String = _ + override def testType = testImpl } \ No newline at end of file diff --git a/test/junit/scala/reflect/internal/names/ExtendedNameTest.scala b/test/junit/scala/reflect/internal/names/ExtendedNameTest.scala index 38cf4521137..f8f7a76cd0b 100644 --- a/test/junit/scala/reflect/internal/names/ExtendedNameTest.scala +++ b/test/junit/scala/reflect/internal/names/ExtendedNameTest.scala @@ -1,25 +1,23 @@ package scala.reflect.internal.names -import java.util -import java.util.concurrent.{CyclicBarrier, Executors, TimeUnit} - import org.junit.Assert._ -import org.junit.Test +import org.junit._ -import scala.concurrent.duration.Duration -import scala.concurrent.{Await, ExecutionContext, Future} -import scala.util.Random +import scala.util.{Allocations, Random} -abstract class ExtendedNameTest { +abstract class ExtendedNameTest extends Allocations { type T <: AnyRef val nameTable: NameTable[T] + def newTermName(s: String) = { nameTable.find(s) } - def newTermName(c:Array[Char], off:Int, len:Int) = { - nameTable.find(c,off,len) + + def newTermName(c: Array[Char], off: Int, len: Int) = { + nameTable.find(c, off, len) } - lazy val sources = Array.tabulate(10000)( i => s"n$i") + + lazy val sources = Array.tabulate(10000)(i => s"n$i") @Test def checkSimple(): Unit = { @@ -29,13 +27,14 @@ abstract class ExtendedNameTest { val n2 = newTermName("xx") assertEquals(1, nameTable.size) - assertSame(n1,n2) + assertSame(n1, n2) val n3 = newTermName(new String("xx")) assertEquals(1, nameTable.size) - assertSame(n1,n3) - assertEquals(n1,n3) + assertSame(n1, n3) + assertEquals(n1, n3) } + @Test def checkByCharsAndByString() { assertEquals(0, nameTable.size) val byString = sources map newTermName @@ -55,6 +54,7 @@ abstract class ExtendedNameTest { } } + @Test def checkByCharsAndByString2() { assertEquals(0, nameTable.size) //same as previous but do the chars first @@ -75,6 +75,7 @@ abstract class ExtendedNameTest { } } + @Test def checkSources1(): Unit = { assertEquals(0, nameTable.size) @@ -86,7 +87,7 @@ abstract class ExtendedNameTest { val shuffled: List[Int] = Random.shuffle(List.tabulate(sources.length)(i => i)) - shuffled.foreach{ + shuffled.foreach { i => val again = newTermName(new String(sources(i))) assertSame(terms(i), again) @@ -99,6 +100,7 @@ abstract class ExtendedNameTest { assertEquals(0, nameTable.size) lookupInRandom } + def lookupInRandom = { val random = new Random() val size = sources.size @@ -106,7 +108,7 @@ abstract class ExtendedNameTest { for (count <- 0 until size * 10) { val i = random.nextInt(size) val term = newTermName(new String(sources(i))) - if (null eq all(i) ) { + if (null eq all(i)) { all(i) = term } else { assertSame(all(i), term) @@ -115,7 +117,7 @@ abstract class ExtendedNameTest { for (i <- 0 until size) { val term = newTermName(new String(sources(i))) - if (null eq all(i) ) { + if (null eq all(i)) { all(i) = term } else { assertSame(all(i), term) @@ -123,5 +125,56 @@ abstract class ExtendedNameTest { } all } + + @Test def lookupByStringNonAllocating: Unit = { + Assume.assumeTrue(nameTable.nonAllocatingStringLookup) + + assertEquals(0, nameTable.size) + val initial = sources map newTermName + + + val byString = { + val res = new Array[AnyRef](sources.length) + assertNonAllocating { + var i = 0 + while (i < sources.length) { + res(i) = newTermName(sources(i)) + i += 1 + } + } + res + } + + + assertEquals(sources.length, nameTable.size) + for (i <- 0 until sources.length) { + assertSame(initial(i), byString(i)) + } + } + + @Test def lookupByCharNonAllocating: Unit = { + Assume.assumeTrue(nameTable.nonAllocatingCharLookup) + assertEquals(0, nameTable.size) + val initial = sources map newTermName + + //warmup any data structures + newTermName("Mike".toCharArray, 0, 4) + + val byChars = { + val res = new Array[AnyRef](sources.length) + assertNonAllocating { + var i = 0 + while (i < sources.length) { + res(i) = newTermName(sources(i).toCharArray, 0, sources(i).length) + i += 1 + } + } + res + } + assertEquals(sources.length, nameTable.size) + for (i <- 0 until sources.length) { + assertSame(initial(i), byChars(i)) + } + } } diff --git a/test/junit/scala/reflect/internal/names/NamesImplTest.scala b/test/junit/scala/reflect/internal/names/NamesImplTest.scala index d2f4d810ab8..6d4a7ca56dd 100644 --- a/test/junit/scala/reflect/internal/names/NamesImplTest.scala +++ b/test/junit/scala/reflect/internal/names/NamesImplTest.scala @@ -6,9 +6,17 @@ class MapNamesTabelTest extends ExtendedNameTest { override type T = Term override val nameTable: NameTable[T] = new MapNameTable[Term](Term.apply) } -class ConcurrentMapNamesTabelTest extends ExtendedNameTest { +class ConcurrentMapNamesTabel1Test extends ExtendedNameTest { override type T = Term - override val nameTable: NameTable[T] = new ConcurrentMapNameTable[Term](Term.apply) + override val nameTable: NameTable[T] = new ConcurrentMapNameTable1[Term](Term.apply) +} +class ConcurrentMapNamesTabel2Test extends ExtendedNameTest { + override type T = Term + override val nameTable: NameTable[T] = new ConcurrentMapNameTable2[Term](Term.apply) +} +class ConcurrentMapNamesTabel3Test extends ExtendedNameTest { + override type T = Term + override val nameTable: NameTable[T] = new ConcurrentMapNameTable3[Term](Term.apply) } class StrongConcurrentNodeInternerTest extends ExtendedNameTest with ConcurrentNamesTest { override type T = Term @@ -18,9 +26,23 @@ class WeakMapNameTableTest extends ExtendedNameTest with WeakNamesTest { override type T = Term override val nameTable: NameTable[T] = new WeakMapNameTable[Term](Term.apply) } -class WeakConcurrentMapNameTableTest extends ExtendedNameTest with WeakNamesTest { +class WeakAutoTrimConcurrentMapNameTable1Test extends ExtendedNameTest with WeakNamesTest { + override type T = Term + override val nameTable: NameTable[T] = new WeakAutoTrimConcurrentMapNameTable1[Term](Term.apply) +} +class WeakAutoTrimConcurrentMapNameTable2Test extends ExtendedNameTest with WeakNamesTest { override type T = Term - override val nameTable: NameTable[T] = new WeakConcurrentMapNameTable[Term](Term.apply) + override val nameTable: NameTable[T] = new WeakAutoTrimConcurrentMapNameTable2[Term](Term.apply) +} +class WeakNoAutoTrimConcurrentMapNameTable1Test extends ExtendedNameTest with WeakNamesTest { + override type T = Term + override val nameTable: WeakNoAutoTrimConcurrentMapNameTable1[T] = new WeakNoAutoTrimConcurrentMapNameTable1[Term](Term.apply) + override def cleanupIfNeeded(): Unit = nameTable.trim() +} +class WeakAutoNoTrimConcurrentMapNameTable2Test extends ExtendedNameTest with WeakNamesTest { + override type T = Term + override val nameTable: WeakNoAutoTrimConcurrentMapNameTable2[T] = new WeakNoAutoTrimConcurrentMapNameTable2[Term](Term.apply) + override def cleanupIfNeeded(): Unit = nameTable.trim() } class WeakFixedSizeNoAutoTrimConcurrentNodeInternerTest extends ExtendedNameTest with WeakNamesTest { override type T = Term @@ -42,6 +64,7 @@ class Find0 extends ExtendedNameTest { class Find1 extends ExtendedNameTest { override type T = Term override val nameTable: NameTable[T] = new WeakFixedSizeAutoTrimConcurrentNodeInterner[Term](Term.apply) { + override def nonAllocatingCharLookup: Boolean = true override def find(chars: Array[Char], start: Int, count: Int): Term = super.find1(chars, start, count) } } @@ -54,6 +77,7 @@ class Find2 extends ExtendedNameTest { class Find3 extends ExtendedNameTest { override type T = Term override val nameTable: NameTable[T] = new WeakFixedSizeAutoTrimConcurrentNodeInterner[Term](Term.apply) { + override def nonAllocatingCharLookup: Boolean = true override def find(chars: Array[Char], start: Int, count: Int): Term = super.find3(chars, start, count) } } diff --git a/test/junit/scala/util/Allocations.scala b/test/junit/scala/util/Allocations.scala new file mode 100644 index 00000000000..a537995c51d --- /dev/null +++ b/test/junit/scala/util/Allocations.scala @@ -0,0 +1,36 @@ +package scala.util + +import scala.tools.nsc.profile.ExtendedThreadMxBean +import org.junit.Assert._ + +object Allocations { + val threadMx = ExtendedThreadMxBean.proxy() + if (threadMx.isThreadAllocatedMemorySupported) threadMx.setThreadAllocatedMemoryEnabled(true) + + //warmup + for (i <- 1 to 100) allocations(i) + //calibrate + val bytes = (((1 to 100).map { i => allocations(i)}).unzip)._2 + assert(bytes.min == bytes.max) + val expected = bytes.head + private def allocations[T]( fn: => T) : (T,Long) = { + require (threadMx.isThreadAllocatedMemoryEnabled, "allocation counting is not enabled") + val currentThread = Thread.currentThread().getId + val before = threadMx.getThreadAllocatedBytes(currentThread) + val res = fn + val after = threadMx.getThreadAllocatedBytes((currentThread)) + (res, after - before - expected) + } + +} +trait Allocations { + import Allocations._ + def assertNonAllocating[T]( fn: => T) : T = { + val (result, bytes) = allocations(fn) + if (bytes != 0) fail (s"expected no allocations but the thread allocated $bytes bytes") + result + } + def resultAndAllocations[T]( fn: => T) : (T, Long) = { + allocations(fn) + } +} From ae00d53c26d3a93b39c977ebba8071623ee66168 Mon Sep 17 00:00:00 2001 From: Jason Zaugg Date: Wed, 26 Jun 2019 16:26:24 +1000 Subject: [PATCH 7/7] Print out estimated retained size of name table after run with -verbose --- src/compiler/scala/tools/nsc/Global.scala | 4 ++++ .../scala/reflect/internal/Names.scala | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/src/compiler/scala/tools/nsc/Global.scala b/src/compiler/scala/tools/nsc/Global.scala index 9bf44d78976..33f524be225 100644 --- a/src/compiler/scala/tools/nsc/Global.scala +++ b/src/compiler/scala/tools/nsc/Global.scala @@ -1554,6 +1554,10 @@ class Global(var currentSettings: Settings, reporter0: Reporter) if (traceSymbolActivity) units map (_.body) foreach (traceSymbols recordSymbolsInTree _) + if (settings.verbose) { + inform(s"[name table stats: ${nameTableStats()}]") + } + // In case no phase was specified for -Xshow-class/object, show it now for sure. if (settings.Yshow.isDefault) showMembers() diff --git a/src/reflect/scala/reflect/internal/Names.scala b/src/reflect/scala/reflect/internal/Names.scala index e74257dde1d..e8159064c5f 100644 --- a/src/reflect/scala/reflect/internal/Names.scala +++ b/src/reflect/scala/reflect/internal/Names.scala @@ -54,6 +54,25 @@ trait Names extends api.Names { final def allNames(): Iterator[TermName] = termHashtable.iterator.filter(_ ne null).flatMap(n => Iterator.iterate(n)(_.next).takeWhile(_ ne null)) + final def nameTableStats(): NameTableStats = { + val numTermNames = termHashtable.count(_ ne null) + val numTypeNames = typeHashtable.count(_ ne null) + val numChars = termHashtable.iterator.filter(_ ne null).map(_.length).sum + val RefSize = 4 + val CharSize = 2 + val TermNameSize = 32 + val TypeNameSize = TermNameSize + val size = (numTermNames + numTypeNames) * RefSize + numChars * CharSize + numTermNames * TermNameSize + numTypeNames * TypeNameSize + val slack = ((termHashtable.length - numTermNames) + (typeHashtable.length - numTypeNames)) * RefSize + (_chrs.length - numChars) * CharSize + new NameTableStats(numTermNames, numTypeNames, size, slack) + } + + final class NameTableStats(val numTermNames: Int, val numTypeNames: Int, val size: Long, val slack: Long) { + override def toString() = { + s"Name table contains ${numTermNames} terms and ${numTypeNames} types consuming $size bytes. There are also $slack bytes of slack space for a total of ${size + slack} bytes." + } + } + private def hashValue(cs: Array[Char], offset: Int, len: Int): Int = { var h = 0 var i = 0