Skip to content

Commit 57dd672

Browse files
Merge pull request #9925 from dotty-staging/scala-library-from-tasty-tests
Compile Scala library with Dotty and test its TASTy
2 parents 70787ae + b02abfc commit 57dd672

File tree

9 files changed

+304
-9
lines changed

9 files changed

+304
-9
lines changed

.github/workflows/ci.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ jobs:
9797

9898
- name: Test
9999
run: |
100-
./project/scripts/sbt ";scala3-bootstrapped/compile ;scala3-bootstrapped/test;sjsSandbox/run;sjsSandbox/test;sjsJUnitTests/test;sjsCompilerTests/test ;sbt-dotty/scripted scala2-compat/* ;configureIDE"
100+
./project/scripts/sbt ";scala3-bootstrapped/compile ;scala3-bootstrapped/test;sjsSandbox/run;sjsSandbox/test;sjsJUnitTests/test;sjsCompilerTests/test ;sbt-dotty/scripted scala2-compat/* ;configureIDE ;stdlib-bootstrapped/test:run ;stdlib-bootstrapped-tasty-tests/test"
101101
./project/scripts/bootstrapCmdTests
102102
103103
## Only run bootstrapped tests for Windows since that's a superset of the

build.sbt

+2
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ val `scala3-tasty-inspector` = Build.`scala3-tasty-inspector`
1515
val `scala3-language-server` = Build.`scala3-language-server`
1616
val `scala3-bench` = Build.`scala3-bench`
1717
val `scala3-bench-bootstrapped` = Build.`scala3-bench-bootstrapped`
18+
val `stdlib-bootstrapped` = Build.`stdlib-bootstrapped`
19+
val `stdlib-bootstrapped-tasty-tests` = Build.`stdlib-bootstrapped-tasty-tests`
1820
val `tasty-core` = Build.`tasty-core`
1921
val `tasty-core-bootstrapped` = Build.`tasty-core-bootstrapped`
2022
val `tasty-core-scala2` = Build.`tasty-core-scala2`

compiler/src/dotty/tools/backend/sjs/JSDefinitions.scala

+1-1
Original file line numberDiff line numberDiff line change
@@ -255,7 +255,7 @@ final class JSDefinitions()(using Context) {
255255
def isJSThisFunctionClass(cls: Symbol): Boolean =
256256
isScalaJSVarArityClass(cls, "ThisFunction")
257257

258-
/** Definitions related to the treatment of JUnit boostrappers. */
258+
/** Definitions related to the treatment of JUnit bootstrappers. */
259259
object junit {
260260
@threadUnsafe lazy val TestAnnotType: TypeRef = requiredClassRef("org.junit.Test")
261261
def TestAnnotClass(using Context): ClassSymbol = TestAnnotType.symbol.asClass

compiler/src/dotty/tools/dotc/core/Symbols.scala

+2-2
Original file line numberDiff line numberDiff line change
@@ -472,8 +472,8 @@ object Symbols {
472472
flags: FlagSet = sym.flags,
473473
info: Type = sym.info,
474474
privateWithin: Symbol = sym.privateWithin,
475-
coord: Coord = NoCoord, // Can be `= owner.coord` once we boostrap
476-
associatedFile: AbstractFile = null // Can be `= owner.associatedFile` once we boostrap
475+
coord: Coord = NoCoord, // Can be `= owner.coord` once we bootstrap
476+
associatedFile: AbstractFile = null // Can be `= owner.associatedFile` once we bootstrap
477477
): Symbol = {
478478
val coord1 = if (coord == NoCoord) owner.coord else coord
479479
val associatedFile1 = if (associatedFile == null) owner.associatedFile else associatedFile

compiler/src/dotty/tools/dotc/transform/sjs/JUnitBootstrappers.scala

+2-2
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ import dotty.tools.backend.sjs.JSDefinitions.jsdefn
9898
* framework with official asynchronous support instead.
9999
*
100100
* Because `Booststrapper` is annotated with `@EnableReflectiveInstantiation`,
101-
* the run-time implementation of JUnit for Scala.js can load the boostrapper
101+
* the run-time implementation of JUnit for Scala.js can load the bootstrapper
102102
* module using `scala.scalajs.reflect.Reflect`, and then use the methods of
103103
* Bootstrapper, which are implemented in the bootstrapper object, to perform
104104
* test discovery and invocation.
@@ -145,7 +145,7 @@ class JUnitBootstrappers extends MiniPhase {
145145
private def genBootstrapper(testClass: ClassSymbol)(using Context): TypeDef = {
146146
val junitdefn = jsdefn.junit
147147

148-
/* The name of the boostrapper module. It is derived from the test class name by
148+
/* The name of the bootstrapper module. It is derived from the test class name by
149149
* appending a specific suffix string mandated "by spec". It will indeed also be
150150
* computed as such at run-time by the Scala.js JUnit Runtime support. Therefore,
151151
* it must *not* be a dotc semantic name.

project/Build.scala

+70-1
Original file line numberDiff line numberDiff line change
@@ -247,7 +247,7 @@ object Build {
247247
excludeFromIDE := true
248248
)
249249

250-
// Settings used when compiling dotty (both non-boostrapped and bootstrapped)
250+
// Settings used when compiling dotty (both non-bootstrapped and bootstrapped)
251251
lazy val commonDottySettings = commonSettings ++ Seq(
252252
// Manually set the standard library to use
253253
autoScalaLibrary := false
@@ -805,6 +805,75 @@ object Build {
805805
javaOptions := (javaOptions in `scala3-compiler-bootstrapped`).value
806806
)
807807

808+
/** Scala library compiled by dotty using the latest published sources of the library */
809+
lazy val `stdlib-bootstrapped` = project.in(file("stdlib-bootstrapped")).
810+
withCommonSettings(Bootstrapped).
811+
dependsOn(dottyCompiler(Bootstrapped) % "provided; compile->runtime; test->test").
812+
dependsOn(`scala3-tasty-inspector` % "test->test").
813+
settings(commonBootstrappedSettings).
814+
settings(
815+
moduleName := "scala-library",
816+
javaOptions := (javaOptions in `scala3-compiler-bootstrapped`).value,
817+
scalacOptions -= "-Xfatal-warnings",
818+
ivyConfigurations += SourceDeps.hide,
819+
transitiveClassifiers := Seq("sources"),
820+
libraryDependencies +=
821+
("org.scala-lang" % "scala-library" % stdlibVersion(Bootstrapped) % "sourcedeps"),
822+
sourceGenerators in Compile += Def.task {
823+
val s = streams.value
824+
val cacheDir = s.cacheDirectory
825+
val trgDir = (sourceManaged in Compile).value / "scala-library-src"
826+
827+
val report = updateClassifiers.value
828+
val scalaLibrarySourcesJar = report.select(
829+
configuration = configurationFilter("sourcedeps"),
830+
module = (_: ModuleID).name == "scala-library",
831+
artifact = artifactFilter(`type` = "src")).headOption.getOrElse {
832+
sys.error(s"Could not fetch scala-library sources")
833+
}
834+
835+
FileFunction.cached(cacheDir / s"fetchScalaLibrarySrc",
836+
FilesInfo.lastModified, FilesInfo.exists) { dependencies =>
837+
s.log.info(s"Unpacking scala-library sources to $trgDir...")
838+
if (trgDir.exists)
839+
IO.delete(trgDir)
840+
IO.createDirectory(trgDir)
841+
IO.unzip(scalaLibrarySourcesJar, trgDir)
842+
843+
((trgDir ** "*.scala") +++ (trgDir ** "*.java")).get.toSet
844+
} (Set(scalaLibrarySourcesJar)).toSeq
845+
}.taskValue,
846+
sources in Compile ~= (_.filterNot(file =>
847+
// sources from https://github.com/scala/scala/tree/2.13.x/src/library-aux
848+
file.getPath.endsWith("scala-library-src/scala/Any.scala") ||
849+
file.getPath.endsWith("scala-library-src/scala/AnyVal.scala") ||
850+
file.getPath.endsWith("scala-library-src/scala/AnyRef.scala") ||
851+
file.getPath.endsWith("scala-library-src/scala/Nothing.scala") ||
852+
file.getPath.endsWith("scala-library-src/scala/Null.scala") ||
853+
file.getPath.endsWith("scala-library-src/scala/Singleton.scala"))),
854+
managedClasspath in Test ~= {
855+
_.filterNot(file => file.data.getName == s"scala-library-${stdlibVersion(Bootstrapped)}.jar")
856+
},
857+
)
858+
859+
/** Test the tasty generated by `stdlib-bootstrapped`
860+
*
861+
* The tests are run with the bootstrapped compiler and the tasty inpector on the classpath.
862+
* The classpath has the default `scala-library` and not `stdlib-bootstrapped`.
863+
*
864+
* The jar of `stdlib-bootstrapped` is provided for to the tests.
865+
* - inspector: test that we can load the contents of the jar using the tasty inspector
866+
* - from-tasty: test that we can recompile the contents of the jar using `dotc -from-tasty`
867+
*/
868+
lazy val `stdlib-bootstrapped-tasty-tests` = project.in(file("stdlib-bootstrapped-tasty-tests")).
869+
withCommonSettings(Bootstrapped).
870+
dependsOn(`scala3-tasty-inspector` % "test->test").
871+
settings(commonBootstrappedSettings).
872+
settings(
873+
javaOptions := (javaOptions in `scala3-compiler-bootstrapped`).value,
874+
javaOptions += "-Ddotty.scala.library=" + packageBin.in(`stdlib-bootstrapped`, Compile).value.getAbsolutePath
875+
)
876+
808877
lazy val `scala3-sbt-bridge` = project.in(file("sbt-bridge/src")).
809878
// We cannot depend on any bootstrapped project to compile the bridge, since the
810879
// bridge is needed to compile these projects.
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,209 @@
1+
package dotty.tools.dotc
2+
3+
import org.junit.Test
4+
import org.junit.Ignore
5+
import org.junit.Assert._
6+
7+
import dotty.tools.io._
8+
import dotty.tools.dotc.util.ClasspathFromClassloader
9+
10+
import scala.quoted._
11+
12+
import java.io.File.pathSeparator
13+
14+
class BootstrappedStdLibTASYyTest:
15+
16+
import BootstrappedStdLibTASYyTest._
17+
18+
/** Test that we can load trees from TASTy */
19+
@Test def testTastyInspector: Unit =
20+
loadWithTastyInspector(loadBlacklisted)
21+
22+
/** Test that we can load and compile trees from TASTy */
23+
@Test def testFromTasty: Unit =
24+
compileFromTasty(loadBlacklisted.union(compileBlacklisted))
25+
26+
@Ignore
27+
@Test def testWhiteListFromTasty: Unit =
28+
val whitelist = Set(
29+
"scala.collection.mutable.StringBuilder"
30+
)
31+
compileFromTasty(x => !whitelist(x))
32+
33+
@Test def blacklistNoDuplicates =
34+
def testDup(name: String, list: List[String], set: Set[String]) =
35+
assert(list.size == set.size,
36+
list.diff(set.toSeq).mkString(s"`$name` has duplicate entries:\n ", "\n ", "\n\n"))
37+
testDup("loadBlacklist", loadBlacklist, loadBlacklisted)
38+
testDup("compileBlacklist", compileBlacklist, compileBlacklisted)
39+
40+
@Test def blacklistsNoIntersection =
41+
val intersection = loadBlacklisted & compileBlacklisted
42+
assert(intersection.isEmpty,
43+
intersection.mkString(
44+
"`compileBlacklist` contains names that are already in `loadBlacklist`: \n ", "\n ", "\n\n"))
45+
46+
@Test def blacklistsOnlyContainsClassesThatExist =
47+
val scalaLibJarTastyClassNamesSet = scalaLibJarTastyClassNames.toSet
48+
val intersection = loadBlacklisted & compileBlacklisted
49+
assert(loadBlacklisted.diff(scalaLibJarTastyClassNamesSet).isEmpty,
50+
loadBlacklisted.diff(scalaLibJarTastyClassNamesSet).mkString(
51+
"`loadBlacklisted` contains names that are not in `scalaLibJarTastyClassNames`: \n ", "\n ", "\n\n"))
52+
assert(compileBlacklisted.diff(scalaLibJarTastyClassNamesSet).isEmpty,
53+
compileBlacklisted.diff(scalaLibJarTastyClassNamesSet).mkString(
54+
"`loadBlacklisted` contains names that are not in `scalaLibJarTastyClassNames`: \n ", "\n ", "\n\n"))
55+
56+
@Ignore
57+
@Test def testLoadBacklistIsMinimal =
58+
var shouldBeWhitelisted = List.empty[String]
59+
val size = loadBlacklisted.size
60+
for (notBlacklisted, i) <- loadBlacklist.zipWithIndex do
61+
val blacklist = loadBlacklisted - notBlacklisted
62+
println(s"Trying withouth $notBlacklisted in the blacklist (${i+1}/$size)")
63+
try {
64+
loadWithTastyInspector(blacklist)
65+
shouldBeWhitelisted = notBlacklisted :: shouldBeWhitelisted
66+
}
67+
catch {
68+
case ex: Throwable => // ok
69+
}
70+
assert(shouldBeWhitelisted.isEmpty,
71+
shouldBeWhitelisted.mkString("Some classes do not need to be blacklisted in `loadBlacklisted`\n ", "\n ", "\n\n"))
72+
73+
@Ignore
74+
@Test def testCompileBlacklistIsMinimal =
75+
var shouldBeWhitelisted = List.empty[String]
76+
val size = compileBlacklisted.size
77+
val blacklist0 = loadBlacklisted.union(compileBlacklisted)
78+
for (notBlacklisted, i) <- compileBlacklist.zipWithIndex do
79+
val blacklist = blacklist0 - notBlacklisted
80+
println(s"Trying withouth $notBlacklisted in the blacklist (${i+1}/$size)")
81+
try {
82+
compileFromTasty(blacklist)
83+
shouldBeWhitelisted = notBlacklisted :: shouldBeWhitelisted
84+
}
85+
catch {
86+
case ex: Throwable => // ok
87+
}
88+
assert(shouldBeWhitelisted.isEmpty,
89+
shouldBeWhitelisted.mkString("Some classes do not need to be blacklisted in `compileBlacklisted`\n ", "\n ", "\n\n"))
90+
91+
end BootstrappedStdLibTASYyTest
92+
93+
object BootstrappedStdLibTASYyTest:
94+
95+
val scalaLibJarPath = System.getProperty("dotty.scala.library")
96+
97+
val scalaLibJarTastyClassNames = {
98+
val scalaLibJar = Jar(new File(java.nio.file.Paths.get(scalaLibJarPath)))
99+
scalaLibJar.toList.map(_.toString).filter(_.endsWith(".tasty"))
100+
.map(_.stripSuffix(".tasty").replace("/", "."))
101+
.sorted
102+
}
103+
104+
def loadWithTastyInspector(blacklisted: String => Boolean): Unit =
105+
val inspector = new scala.tasty.inspector.TastyInspector {
106+
def processCompilationUnit(using QuoteContext)(root: qctx.reflect.Tree): Unit =
107+
root.showExtractors // Check that we can traverse the full tree
108+
()
109+
}
110+
val classNames = scalaLibJarTastyClassNames.filterNot(blacklisted)
111+
val hasErrors = inspector.inspect(scalaLibJarPath, classNames)
112+
assert(!hasErrors, "Errors reported while loading from TASTy")
113+
114+
def compileFromTasty(blacklisted: String => Boolean): Unit = {
115+
val driver = new dotty.tools.dotc.Driver
116+
val currentClasspath = ClasspathFromClassloader(getClass.getClassLoader)
117+
val classNames = scalaLibJarTastyClassNames.filterNot(blacklisted)
118+
val args = Array(
119+
"-classpath", s"$scalaLibJarPath$pathSeparator$currentClasspath",
120+
"-from-tasty",
121+
"-nowarn"
122+
) ++ classNames
123+
val reporter = driver.process(args)
124+
assert(reporter.errorCount == 0, "Errors while re-compiling")
125+
}
126+
127+
/** List of classes that cannot be loaded from TASTy */
128+
def loadBlacklist = List[String](
129+
// No issues :)
130+
)
131+
132+
/** List of classes that cannot be recompilied from TASTy */
133+
def compileBlacklist = List[String](
134+
// See #10048
135+
// failed: java.lang.AssertionError: assertion failed: class Boolean
136+
// at dotty.DottyPredef$.assertFail(DottyPredef.scala:17)
137+
// at dotty.tools.backend.jvm.BCodeHelpers$BCInnerClassGen.assertClassNotArrayNotPrimitive(BCodeHelpers.scala:247)
138+
// at dotty.tools.backend.jvm.BCodeHelpers$BCInnerClassGen.getClassBTypeAndRegisterInnerClass(BCodeHelpers.scala:265)
139+
// at dotty.tools.backend.jvm.BCodeHelpers$BCInnerClassGen.getClassBTypeAndRegisterInnerClass$(BCodeHelpers.scala:210)
140+
// at dotty.tools.backend.jvm.BCodeSkelBuilder$PlainSkelBuilder.getClassBTypeAndRegisterInnerClass(BCodeSkelBuilder.scala:62)
141+
// at dotty.tools.backend.jvm.BCodeHelpers$BCInnerClassGen.internalName(BCodeHelpers.scala:237)
142+
"scala.Array",
143+
"scala.Boolean",
144+
"scala.Byte",
145+
"scala.Char",
146+
"scala.Double",
147+
"scala.Float",
148+
"scala.Int",
149+
"scala.Long",
150+
"scala.Short",
151+
"scala.Unit",
152+
153+
// See #9994
154+
// -- Error:
155+
// | def addOne(kv: (K, V)) = {
156+
// | ^
157+
// |error overriding method addOne in trait Growable of type (elem: (K, V)): (TrieMap.this : scala.collection.concurrent.TrieMap[K, V]);
158+
// | method addOne of type (kv: (K, V)): (TrieMap.this : scala.collection.concurrent.TrieMap[K, V]) has incompatible type
159+
// -- Error:
160+
// | def subtractOne(k: K) = {
161+
// | ^
162+
// |error overriding method subtractOne in trait Shrinkable of type (elem: K): (TrieMap.this : scala.collection.concurrent.TrieMap[K, V]);
163+
// | method subtractOne of type (k: K): (TrieMap.this : scala.collection.concurrent.TrieMap[K, V]) has incompatible type
164+
"scala.collection.concurrent.TrieMap",
165+
"scala.collection.immutable.HashMapBuilder",
166+
"scala.collection.immutable.HashSetBuilder",
167+
"scala.collection.immutable.LazyList",
168+
"scala.collection.immutable.ListMapBuilder",
169+
"scala.collection.immutable.MapBuilderImpl",
170+
"scala.collection.immutable.SetBuilderImpl",
171+
"scala.collection.immutable.TreeSeqMap",
172+
"scala.collection.immutable.VectorBuilder",
173+
"scala.collection.immutable.VectorMapBuilder",
174+
"scala.collection.mutable.AnyRefMap",
175+
"scala.collection.mutable.ArrayBuilder",
176+
"scala.collection.mutable.CollisionProofHashMap",
177+
"scala.collection.mutable.LongMap",
178+
"scala.collection.mutable.SortedMap",
179+
"scala.collection.mutable.StringBuilder",
180+
"scala.jdk.AnyAccumulator",
181+
"scala.jdk.DoubleAccumulator",
182+
"scala.jdk.IntAccumulator",
183+
"scala.jdk.LongAccumulator",
184+
185+
// See #9994
186+
// -- Error:
187+
// | override def filterInPlace(p: A => Boolean): this.type = {
188+
// | ^
189+
// |error overriding method filterInPlace in trait SetOps of type (p: A => Boolean): (HashSet.this : scala.collection.mutable.HashSet[A]);
190+
// | method filterInPlace of type (p: A => Boolean): (HashSet.this : scala.collection.mutable.HashSet[A]) has incompatible type
191+
"scala.collection.mutable.HashSet",
192+
193+
// See #9994
194+
// -- Error:
195+
// | def force: this.type = {
196+
// | ^
197+
// |error overriding method force in class Stream of type => (Cons.this : scala.collection.immutable.Stream.Cons[A]);
198+
// | method force of type => (Cons.this : scala.collection.immutable.Stream.Cons[A]) has incompatible type
199+
"scala.collection.immutable.Stream",
200+
201+
)
202+
203+
/** Set of classes that cannot be loaded from TASTy */
204+
def loadBlacklisted = loadBlacklist.toSet
205+
206+
/** Set of classes that cannot be recompilied from TASTy */
207+
def compileBlacklisted = compileBlacklist.toSet
208+
209+
end BootstrappedStdLibTASYyTest

stdlib-bootstrapped/test/Main.scala

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
package hello
2+
3+
enum Color:
4+
case Red, Green, Blue
5+
6+
object HelloWorld:
7+
def main(args: Array[String]): Unit = {
8+
println("hello dotty.superbootstrapped!")
9+
println(Color.Red)
10+
println(Color.Green)
11+
println(Color.Blue)
12+
}

tasty-inspector/src/scala/tasty/inspector/TastyInspector.scala

+5-2
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,9 @@ trait TastyInspector:
2424
*
2525
* @param classpath Classpath where the classes are located
2626
* @param classes classes to be inspected
27+
* @return if an error was reported
2728
*/
28-
def inspect(classpath: String, classes: List[String]): Unit =
29+
def inspect(classpath: String, classes: List[String]): Boolean =
2930
if (classes.isEmpty)
3031
throw new IllegalArgumentException("Parameter classes should no be empty")
3132

@@ -64,7 +65,9 @@ trait TastyInspector:
6465

6566
val currentClasspath = ClasspathFromClassloader(getClass.getClassLoader)
6667
val args = "-from-tasty" :: "-Yretain-trees" :: "-classpath" :: s"$classpath$pathSeparator$currentClasspath" :: classes
67-
(new InspectorDriver).process(args.toArray)
68+
val reporter = (new InspectorDriver).process(args.toArray)
69+
reporter.hasErrors
70+
6871
end inspect
6972

7073

0 commit comments

Comments
 (0)