Skip to content

Commit a39d405

Browse files
prolativprolativ
committed
Improve bisect script - speed up bisection and improve bisection procedure for most common bisection types
1 parent d15e2bd commit a39d405

File tree

3 files changed

+194
-113
lines changed

3 files changed

+194
-113
lines changed

‎project/scripts/bisect.scala

+192-111
Original file line numberDiff line numberDiff line change
@@ -8,98 +8,203 @@ Look at the `usageMessage` below for more details.
88

99
import sys.process._
1010
import scala.io.Source
11-
import Releases.Release
1211
import java.io.File
12+
import java.nio.file.attribute.PosixFilePermissions
13+
import java.nio.charset.StandardCharsets
14+
import java.nio.file.Files
1315

1416
val usageMessage = """
1517
|Usage:
16-
| > scala-cli project/scripts/bisect.scala -- <validation-script-path> [versions-range]
18+
| > scala-cli project/scripts/bisect.scala -- [<bisect-options>] <validation-command>
1719
|
18-
|The validation script should be executable and accept a single parameter, which will be the scala version to validate.
20+
|The <validation-command> should be one of:
21+
|* compile <arg1> <arg2> ...
22+
|* run <arg1> <arg2> ...
23+
|* <custom-validation-script-path>
24+
|
25+
|The arguments for 'compile' and 'run' should be paths to the source file(s) and optionally additional options passed directly to scala-cli.
26+
|
27+
|A custom validation script should be executable and accept a single parameter, which will be the scala version to validate.
1928
|Look at bisect-cli-example.sh and bisect-expect-example.exp for reference.
20-
|The optional versions range specifies which releases should be taken into account while bisecting.
21-
|The range format is <first>...<last>, where both <first> and <last> are optional, e.g.
22-
|* 3.1.0-RC1-bin-20210827-427d313-NIGHTLY..3.2.1-RC1-bin-20220716-bb9c8ff-NIGHTLY
23-
|* 3.2.1-RC1-bin-20220620-de3a82c-NIGHTLY..
24-
|* ..3.3.0-RC1-bin-20221124-e25362d-NIGHTLY
25-
|The ranges are treated as inclusive.
29+
|If you want to use one of the example scripts - use a copy of the file instead of modifying it in place because that might mess up the checkout.
2630
|
27-
|Don't use the example scripts modified in place as they might disappear from the repo during a checkout.
28-
|Instead copy them to a different location first.
31+
|The optional <bisect-options> may be any combination of:
32+
|* --dry-run
33+
| Don't try to bisect - just make sure the validation command works correctly
34+
|* --releases <releases-range>
35+
| Bisect only releases from the given range (defaults to all releases).
36+
| The range format is <first>...<last>, where both <first> and <last> are optional, e.g.
37+
| * 3.1.0-RC1-bin-20210827-427d313-NIGHTLY..3.2.1-RC1-bin-20220716-bb9c8ff-NIGHTLY
38+
| * 3.2.1-RC1-bin-20220620-de3a82c-NIGHTLY..
39+
| * ..3.3.0-RC1-bin-20221124-e25362d-NIGHTLY
40+
| The ranges are treated as inclusive.
41+
|* --bootstrapped
42+
| Publish locally and test a bootstrapped compiler rather than a nonboostrapped one
2943
|
3044
|Warning: The bisect script should not be run multiple times in parallel because of a potential race condition while publishing artifacts locally.
31-
|
32-
|Tip: Before running the bisect script run the validation script manually with some published versions of the compiler to make sure it succeeds and fails as expected.
45+
3346
""".stripMargin
3447

35-
@main def dottyCompileBisect(args: String*): Unit =
36-
val (validationScriptRawPath, versionsRange) = args match
37-
case Seq(path) =>
38-
(path, VersionsRange.all)
39-
case Seq(path, ParsedVersionsRange(range)) =>
40-
(path, range)
41-
case _ =>
42-
println("Wrong script parameters.")
43-
println()
44-
println(usageMessage)
45-
System.exit(1)
46-
null
47-
48-
val validationScriptPath = (new File(validationScriptRawPath)).getAbsolutePath.toString
49-
given releases: Releases = Releases.fromRange(versionsRange)
50-
51-
val releaseBisect = ReleaseBisect(validationScriptPath)
52-
val bisectedBadRelease = releaseBisect.bisectedBadRelease(releases.releases)
53-
println("\nFinished bisecting releases\n")
54-
55-
bisectedBadRelease match
56-
case Some(firstBadRelease) =>
57-
firstBadRelease.previous match
58-
case Some(lastGoodRelease) =>
59-
println(s"Last good release: $lastGoodRelease")
60-
println(s"First bad release: $firstBadRelease")
61-
val commitBisect = CommitBisect(validationScriptPath)
62-
commitBisect.bisect(lastGoodRelease.hash, firstBadRelease.hash)
63-
case None =>
64-
println(s"No good release found")
65-
case None =>
66-
println(s"No bad release found")
67-
68-
case class VersionsRange(first: Option[String], last: Option[String]):
69-
def filter(versions: Seq[String]) =
70-
def versionIndex(version: String) =
71-
val lastMatchingNightly =
72-
if version.contains("-bin-") then version else
73-
versions.filter(_.startsWith(version)).last
74-
versions.indexOf(lastMatchingNightly)
75-
76-
val startIdx = first.map(versionIndex(_)).getOrElse(0)
77-
assert(startIdx >= 0, s"${first} is not a nightly compiler release")
78-
val endIdx = last.map(versionIndex(_) + 1).getOrElse(versions.length)
79-
assert(endIdx > 0, s"${endIdx} is not a nightly compiler release")
80-
val filtered = versions.slice(startIdx, endIdx).toVector
48+
@main def run(args: String*): Unit =
49+
val scriptOptions =
50+
try ScriptOptions.fromArgs(args)
51+
catch
52+
case _ =>
53+
sys.error(s"Wrong script parameters.\n${usageMessage}")
54+
55+
val validationScript = scriptOptions.validationCommand.validationScript
56+
val releases = Releases.fromRange(scriptOptions.releasesRange)
57+
val releaseBisect = ReleaseBisect(validationScript, releases)
58+
59+
releaseBisect.verifyEdgeReleases()
60+
61+
if (!scriptOptions.dryRun) then
62+
val (lastGoodRelease, firstBadRelease) = releaseBisect.bisectedGoodAndBadReleases()
63+
println(s"Last good release: ${lastGoodRelease.version}")
64+
println(s"First bad release: ${firstBadRelease.version}")
65+
println("\nFinished bisecting releases\n")
66+
67+
val commitBisect = CommitBisect(validationScript, bootstrapped = scriptOptions.bootstrapped, lastGoodRelease.hash, firstBadRelease.hash)
68+
commitBisect.bisect()
69+
70+
71+
case class ScriptOptions(validationCommand: ValidationCommand, dryRun: Boolean, bootstrapped: Boolean, releasesRange: ReleasesRange)
72+
object ScriptOptions:
73+
def fromArgs(args: Seq[String]) =
74+
val defaultOptions = ScriptOptions(
75+
validationCommand = null,
76+
dryRun = false,
77+
bootstrapped = false,
78+
ReleasesRange(first = None, last = None)
79+
)
80+
parseArgs(args, defaultOptions)
81+
82+
private def parseArgs(args: Seq[String], options: ScriptOptions): ScriptOptions =
83+
args match
84+
case "--dry-run" :: argsRest => parseArgs(argsRest, options.copy(dryRun = true))
85+
case "--bootstrapped" :: argsRest => parseArgs(argsRest, options.copy(bootstrapped = true))
86+
case "--releases" :: argsRest =>
87+
val range = ReleasesRange.tryParse(argsRest.head).get
88+
parseArgs(argsRest.tail, options.copy(releasesRange = range))
89+
case _ =>
90+
val command = ValidationCommand.fromArgs(args)
91+
options.copy(validationCommand = command)
92+
93+
enum ValidationCommand:
94+
case Compile(args: Seq[String])
95+
case Run(args: Seq[String])
96+
case CustomValidationScript(scriptFile: File)
97+
98+
def validationScript: File = this match
99+
case Compile(args) =>
100+
ValidationScript.tmpScalaCliScript(command = "compile", args)
101+
case Run(args) =>
102+
ValidationScript.tmpScalaCliScript(command = "run", args)
103+
case CustomValidationScript(scriptFile) =>
104+
ValidationScript.copiedFrom(scriptFile)
105+
106+
object ValidationCommand:
107+
def fromArgs(args: Seq[String]) = args match
108+
case Seq("compile", commandArgs*) => Compile(commandArgs)
109+
case Seq("run", commandArgs*) => Run(commandArgs)
110+
case Seq(path) => CustomValidationScript(new File(path))
111+
112+
113+
object ValidationScript:
114+
def copiedFrom(file: File): File =
115+
val fileContent = scala.io.Source.fromFile(file).mkString
116+
tmpScript(fileContent)
117+
118+
def tmpScalaCliScript(command: String, args: Seq[String]): File = tmpScript(s"""
119+
|#!/usr/bin/env bash
120+
|scala-cli ${command} -S "$$1" --server=false ${args.mkString(" ")}
121+
|""".stripMargin
122+
)
123+
124+
private def tmpScript(content: String): File =
125+
val executableAttr = PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwxr-xr-x"))
126+
val tmpPath = Files.createTempFile("scala-bisect-validator", "", executableAttr)
127+
val tmpFile = tmpPath.toFile
128+
129+
print(s"Bisecting with validation script: ${tmpPath.toAbsolutePath}\n")
130+
print("#####################################\n")
131+
print(s"${content}\n\n")
132+
print("#####################################\n\n")
133+
134+
tmpFile.deleteOnExit()
135+
Files.write(tmpPath, content.getBytes(StandardCharsets.UTF_8))
136+
tmpFile
137+
138+
139+
case class ReleasesRange(first: Option[String], last: Option[String]):
140+
def filter(releases: Seq[Release]) =
141+
def releaseIndex(version: String): Int =
142+
val index = releases.indexWhere(_.version == version)
143+
assert(index > 0, s"${version} matches no nightly compiler release")
144+
index
145+
146+
val startIdx = first.map(releaseIndex(_)).getOrElse(0)
147+
val endIdx = last.map(releaseIndex(_) + 1).getOrElse(releases.length)
148+
val filtered = releases.slice(startIdx, endIdx).toVector
81149
assert(filtered.nonEmpty, "No matching releases")
82150
filtered
83151

84-
85-
object VersionsRange:
86-
def all = VersionsRange(None, None)
87-
88-
object ParsedVersionsRange:
89-
def unapply(range: String): Option[VersionsRange] = range match
90-
case s"${first}...${last}" => Some(VersionsRange(
152+
object ReleasesRange:
153+
def all = ReleasesRange(None, None)
154+
def tryParse(range: String): Option[ReleasesRange] = range match
155+
case s"${first}...${last}" => Some(ReleasesRange(
91156
Some(first).filter(_.nonEmpty),
92157
Some(last).filter(_.nonEmpty)
93158
))
94159
case _ => None
95160

96-
class ReleaseBisect(validationScriptPath: String):
97-
def bisectedBadRelease(releases: Vector[Release]): Option[Release] =
98-
Some(bisect(releases))
99-
.filter(!isGoodRelease(_))
161+
class Releases(val releases: Vector[Release])
162+
163+
object Releases:
164+
lazy val allReleases: Vector[Release] =
165+
val re = raw"""(?<=title=")(.+-bin-\d{8}-\w{7}-NIGHTLY)(?=/")""".r
166+
val html = Source.fromURL("https://repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/")
167+
re.findAllIn(html.mkString).map(Release.apply).toVector
168+
169+
def fromRange(range: ReleasesRange): Vector[Release] = range.filter(allReleases)
170+
171+
case class Release(version: String):
172+
private val re = raw".+-bin-(\d{8})-(\w{7})-NIGHTLY".r
173+
def date: String =
174+
version match
175+
case re(date, _) => date
176+
case _ => sys.error(s"Could not extract date from release name: $version")
177+
def hash: String =
178+
version match
179+
case re(_, hash) => hash
180+
case _ => sys.error(s"Could not extract hash from release name: $version")
100181

101-
def bisect(releases: Vector[Release]): Release =
102-
assert(releases.length > 1, "Need at least 2 releases to bisect")
182+
override def toString: String = version
183+
184+
185+
class ReleaseBisect(validationScript: File, allReleases: Vector[Release]):
186+
assert(allReleases.length > 1, "Need at least 2 releases to bisect")
187+
188+
private val isGoodReleaseCache = collection.mutable.Map.empty[Release, Boolean]
189+
190+
def verifyEdgeReleases(): Unit =
191+
println(s"Verifying the first release: ${allReleases.head.version}")
192+
assert(isGoodRelease(allReleases.head), s"The evaluation script unexpectedly failed for the first checked release")
193+
println(s"Verifying the last release: ${allReleases.last.version}")
194+
assert(!isGoodRelease(allReleases.last), s"The evaluation script unexpectedly succeeded for the last checked release")
195+
196+
def bisectedGoodAndBadReleases(): (Release, Release) =
197+
val firstBadRelease = bisect(allReleases)
198+
assert(!isGoodRelease(firstBadRelease), s"Bisection error: the 'first bad release' ${firstBadRelease.version} is not a bad release")
199+
val lastGoodRelease = firstBadRelease.previous
200+
assert(isGoodRelease(lastGoodRelease), s"Bisection error: the 'last good release' ${lastGoodRelease.version} is not a good release")
201+
(lastGoodRelease, firstBadRelease)
202+
203+
extension (release: Release) private def previous: Release =
204+
val idx = allReleases.indexOf(release)
205+
allReleases(idx - 1)
206+
207+
private def bisect(releases: Vector[Release]): Release =
103208
if releases.length == 2 then
104209
if isGoodRelease(releases.head) then releases.last
105210
else releases.head
@@ -109,48 +214,24 @@ class ReleaseBisect(validationScriptPath: String):
109214
else bisect(releases.take(releases.length / 2 + 1))
110215

111216
private def isGoodRelease(release: Release): Boolean =
112-
println(s"Testing ${release.version}")
113-
val result = Seq(validationScriptPath, release.version).!
114-
val isGood = result == 0
115-
println(s"Test result: ${release.version} is a ${if isGood then "good" else "bad"} release\n")
116-
isGood
117-
118-
class Releases(val releases: Vector[Release])
119-
120-
object Releases:
121-
private lazy val allReleases: Vector[String] =
122-
val re = raw"""(?<=title=")(.+-bin-\d{8}-\w{7}-NIGHTLY)(?=/")""".r
123-
val html = Source.fromURL("https://repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/")
124-
re.findAllIn(html.mkString).toVector
125-
126-
def fromRange(range: VersionsRange): Releases = Releases(range.filter(allReleases).map(Release(_)))
127-
128-
case class Release(version: String):
129-
private val re = raw".+-bin-(\d{8})-(\w{7})-NIGHTLY".r
130-
def date: String =
131-
version match
132-
case re(date, _) => date
133-
case _ => sys.error(s"Could not extract date from version $version")
134-
def hash: String =
135-
version match
136-
case re(_, hash) => hash
137-
case _ => sys.error(s"Could not extract hash from version $version")
138-
139-
def previous(using r: Releases): Option[Release] =
140-
val idx = r.releases.indexOf(this)
141-
if idx == 0 then None
142-
else Some(r.releases(idx - 1))
143-
144-
override def toString: String = version
145-
146-
class CommitBisect(validationScriptPath: String):
147-
def bisect(lastGoodHash: String, fistBadHash: String): Unit =
217+
isGoodReleaseCache.getOrElseUpdate(release, {
218+
println(s"Testing ${release.version}")
219+
val result = Seq(validationScript.getAbsolutePath, release.version).!
220+
val isGood = result == 0
221+
println(s"Test result: ${release.version} is a ${if isGood then "good" else "bad"} release\n")
222+
isGood
223+
})
224+
225+
class CommitBisect(validationScript: File, bootstrapped: Boolean, lastGoodHash: String, fistBadHash: String):
226+
def bisect(): Unit =
148227
println(s"Starting bisecting commits $lastGoodHash..$fistBadHash\n")
228+
val scala3CompilerProject = if bootstrapped then "scala3-compiler-bootstrapped" else "scala3-compiler"
229+
val scala3Project = if bootstrapped then "scala3-bootstrapped" else "scala3"
149230
val bisectRunScript = s"""
150-
|scalaVersion=$$(sbt "print scala3-compiler-bootstrapped/version" | tail -n1)
231+
|scalaVersion=$$(sbt "print ${scala3CompilerProject}/version" | tail -n1)
151232
|rm -r out
152-
|sbt "clean; scala3-bootstrapped/publishLocal"
153-
|$validationScriptPath "$$scalaVersion"
233+
|sbt "clean; ${scala3Project}/publishLocal"
234+
|${validationScript.getAbsolutePath} "$$scalaVersion"
154235
""".stripMargin
155236
"git bisect start".!
156237
s"git bisect bad $fistBadHash".!

‎project/scripts/examples/bisect-cli-example.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,4 @@
33
# Don't use this example script modified in place as it might disappear from the repo during a checkout.
44
# Instead copy it to a different location first.
55

6-
scala-cli compile -S "$1" file1.scala file2.scala
6+
scala-cli compile -S "$1" --server=false file1.scala file2.scala

‎project/scripts/examples/bisect-expect-example.exp

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
set scalaVersion [lindex $argv 0] ;# Get the script argument
77

88
set timeout 30 ;# Give scala-cli some time to download the compiler
9-
spawn scala-cli repl -S "$scalaVersion" ;# Start the REPL
9+
spawn scala-cli repl -S "$scalaVersion" --server=false ;# Start the REPL
1010
expect "scala>" ;# REPL has started
1111
set timeout 5
1212
send -- "Seq.empty.len\t" ;# Tab pressed to trigger code completion

0 commit comments

Comments
 (0)