From d15e2bdeeafb1d6d64925a930dcf533801fd8bfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pa=C5=82ka?= Date: Tue, 13 Dec 2022 14:25:20 +0100 Subject: [PATCH 1/2] Improve bisect script - bisect releases from a given range --- project/scripts/bisect.scala | 66 ++++++++++++++++++++++++++++++------ 1 file changed, 55 insertions(+), 11 deletions(-) diff --git a/project/scripts/bisect.scala b/project/scripts/bisect.scala index 56b56f71b7fa..ea17a2da740a 100755 --- a/project/scripts/bisect.scala +++ b/project/scripts/bisect.scala @@ -13,10 +13,17 @@ import java.io.File val usageMessage = """ |Usage: - | > scala-cli project/scripts/bisect.scala -- + | > scala-cli project/scripts/bisect.scala -- [versions-range] | |The validation script should be executable and accept a single parameter, which will be the scala version to validate. |Look at bisect-cli-example.sh and bisect-expect-example.exp for reference. + |The optional versions range specifies which releases should be taken into account while bisecting. + |The range format is ..., where both and are optional, e.g. + |* 3.1.0-RC1-bin-20210827-427d313-NIGHTLY..3.2.1-RC1-bin-20220716-bb9c8ff-NIGHTLY + |* 3.2.1-RC1-bin-20220620-de3a82c-NIGHTLY.. + |* ..3.3.0-RC1-bin-20221124-e25362d-NIGHTLY + |The ranges are treated as inclusive. + | |Don't use the example scripts modified in place as they might disappear from the repo during a checkout. |Instead copy them to a different location first. | @@ -26,9 +33,11 @@ val usageMessage = """ """.stripMargin @main def dottyCompileBisect(args: String*): Unit = - val validationScriptPath = args match + val (validationScriptRawPath, versionsRange) = args match case Seq(path) => - (new File(path)).getAbsolutePath.toString + (path, VersionsRange.all) + case Seq(path, ParsedVersionsRange(range)) => + (path, range) case _ => println("Wrong script parameters.") println() @@ -36,8 +45,11 @@ val usageMessage = """ System.exit(1) null + val validationScriptPath = (new File(validationScriptRawPath)).getAbsolutePath.toString + given releases: Releases = Releases.fromRange(versionsRange) + val releaseBisect = ReleaseBisect(validationScriptPath) - val bisectedBadRelease = releaseBisect.bisectedBadRelease(Releases.allReleases) + val bisectedBadRelease = releaseBisect.bisectedBadRelease(releases.releases) println("\nFinished bisecting releases\n") bisectedBadRelease match @@ -53,9 +65,37 @@ val usageMessage = """ case None => println(s"No bad release found") +case class VersionsRange(first: Option[String], last: Option[String]): + def filter(versions: Seq[String]) = + def versionIndex(version: String) = + val lastMatchingNightly = + if version.contains("-bin-") then version else + versions.filter(_.startsWith(version)).last + versions.indexOf(lastMatchingNightly) + + val startIdx = first.map(versionIndex(_)).getOrElse(0) + assert(startIdx >= 0, s"${first} is not a nightly compiler release") + val endIdx = last.map(versionIndex(_) + 1).getOrElse(versions.length) + assert(endIdx > 0, s"${endIdx} is not a nightly compiler release") + val filtered = versions.slice(startIdx, endIdx).toVector + assert(filtered.nonEmpty, "No matching releases") + filtered + + +object VersionsRange: + def all = VersionsRange(None, None) + +object ParsedVersionsRange: + def unapply(range: String): Option[VersionsRange] = range match + case s"${first}...${last}" => Some(VersionsRange( + Some(first).filter(_.nonEmpty), + Some(last).filter(_.nonEmpty) + )) + case _ => None + class ReleaseBisect(validationScriptPath: String): def bisectedBadRelease(releases: Vector[Release]): Option[Release] = - Some(bisect(releases: Vector[Release])) + Some(bisect(releases)) .filter(!isGoodRelease(_)) def bisect(releases: Vector[Release]): Release = @@ -75,11 +115,15 @@ class ReleaseBisect(validationScriptPath: String): println(s"Test result: ${release.version} is a ${if isGood then "good" else "bad"} release\n") isGood +class Releases(val releases: Vector[Release]) + object Releases: - lazy val allReleases: Vector[Release] = - val re = raw"(?<=title=$")(.+-bin-\d{8}-\w{7}-NIGHTLY)(?=/$")".r + private lazy val allReleases: Vector[String] = + val re = raw"""(?<=title=")(.+-bin-\d{8}-\w{7}-NIGHTLY)(?=/")""".r val html = Source.fromURL("https://repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/") - re.findAllIn(html.mkString).map(Release.apply).toVector + re.findAllIn(html.mkString).toVector + + def fromRange(range: VersionsRange): Releases = Releases(range.filter(allReleases).map(Release(_))) case class Release(version: String): private val re = raw".+-bin-(\d{8})-(\w{7})-NIGHTLY".r @@ -92,10 +136,10 @@ object Releases: case re(_, hash) => hash case _ => sys.error(s"Could not extract hash from version $version") - def previous: Option[Release] = - val idx = allReleases.indexOf(this) + def previous(using r: Releases): Option[Release] = + val idx = r.releases.indexOf(this) if idx == 0 then None - else Some(allReleases(idx - 1)) + else Some(r.releases(idx - 1)) override def toString: String = version From a39d40563592c418295c8b45c7b5a6828063ba4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pa=C5=82ka?= Date: Tue, 27 Dec 2022 18:15:58 +0100 Subject: [PATCH 2/2] Improve bisect script - speed up bisection and improve bisection procedure for most common bisection types --- project/scripts/bisect.scala | 303 +++++++++++------- .../scripts/examples/bisect-cli-example.sh | 2 +- .../examples/bisect-expect-example.exp | 2 +- 3 files changed, 194 insertions(+), 113 deletions(-) diff --git a/project/scripts/bisect.scala b/project/scripts/bisect.scala index ea17a2da740a..e60e632a7feb 100755 --- a/project/scripts/bisect.scala +++ b/project/scripts/bisect.scala @@ -8,98 +8,203 @@ Look at the `usageMessage` below for more details. import sys.process._ import scala.io.Source -import Releases.Release import java.io.File +import java.nio.file.attribute.PosixFilePermissions +import java.nio.charset.StandardCharsets +import java.nio.file.Files val usageMessage = """ |Usage: - | > scala-cli project/scripts/bisect.scala -- [versions-range] + | > scala-cli project/scripts/bisect.scala -- [] | - |The validation script should be executable and accept a single parameter, which will be the scala version to validate. + |The should be one of: + |* compile ... + |* run ... + |* + | + |The arguments for 'compile' and 'run' should be paths to the source file(s) and optionally additional options passed directly to scala-cli. + | + |A custom validation script should be executable and accept a single parameter, which will be the scala version to validate. |Look at bisect-cli-example.sh and bisect-expect-example.exp for reference. - |The optional versions range specifies which releases should be taken into account while bisecting. - |The range format is ..., where both and are optional, e.g. - |* 3.1.0-RC1-bin-20210827-427d313-NIGHTLY..3.2.1-RC1-bin-20220716-bb9c8ff-NIGHTLY - |* 3.2.1-RC1-bin-20220620-de3a82c-NIGHTLY.. - |* ..3.3.0-RC1-bin-20221124-e25362d-NIGHTLY - |The ranges are treated as inclusive. + |If you want to use one of the example scripts - use a copy of the file instead of modifying it in place because that might mess up the checkout. | - |Don't use the example scripts modified in place as they might disappear from the repo during a checkout. - |Instead copy them to a different location first. + |The optional may be any combination of: + |* --dry-run + | Don't try to bisect - just make sure the validation command works correctly + |* --releases + | Bisect only releases from the given range (defaults to all releases). + | The range format is ..., where both and are optional, e.g. + | * 3.1.0-RC1-bin-20210827-427d313-NIGHTLY..3.2.1-RC1-bin-20220716-bb9c8ff-NIGHTLY + | * 3.2.1-RC1-bin-20220620-de3a82c-NIGHTLY.. + | * ..3.3.0-RC1-bin-20221124-e25362d-NIGHTLY + | The ranges are treated as inclusive. + |* --bootstrapped + | Publish locally and test a bootstrapped compiler rather than a nonboostrapped one | |Warning: The bisect script should not be run multiple times in parallel because of a potential race condition while publishing artifacts locally. - | - |Tip: Before running the bisect script run the validation script manually with some published versions of the compiler to make sure it succeeds and fails as expected. + """.stripMargin -@main def dottyCompileBisect(args: String*): Unit = - val (validationScriptRawPath, versionsRange) = args match - case Seq(path) => - (path, VersionsRange.all) - case Seq(path, ParsedVersionsRange(range)) => - (path, range) - case _ => - println("Wrong script parameters.") - println() - println(usageMessage) - System.exit(1) - null - - val validationScriptPath = (new File(validationScriptRawPath)).getAbsolutePath.toString - given releases: Releases = Releases.fromRange(versionsRange) - - val releaseBisect = ReleaseBisect(validationScriptPath) - val bisectedBadRelease = releaseBisect.bisectedBadRelease(releases.releases) - println("\nFinished bisecting releases\n") - - bisectedBadRelease match - case Some(firstBadRelease) => - firstBadRelease.previous match - case Some(lastGoodRelease) => - println(s"Last good release: $lastGoodRelease") - println(s"First bad release: $firstBadRelease") - val commitBisect = CommitBisect(validationScriptPath) - commitBisect.bisect(lastGoodRelease.hash, firstBadRelease.hash) - case None => - println(s"No good release found") - case None => - println(s"No bad release found") - -case class VersionsRange(first: Option[String], last: Option[String]): - def filter(versions: Seq[String]) = - def versionIndex(version: String) = - val lastMatchingNightly = - if version.contains("-bin-") then version else - versions.filter(_.startsWith(version)).last - versions.indexOf(lastMatchingNightly) - - val startIdx = first.map(versionIndex(_)).getOrElse(0) - assert(startIdx >= 0, s"${first} is not a nightly compiler release") - val endIdx = last.map(versionIndex(_) + 1).getOrElse(versions.length) - assert(endIdx > 0, s"${endIdx} is not a nightly compiler release") - val filtered = versions.slice(startIdx, endIdx).toVector +@main def run(args: String*): Unit = + val scriptOptions = + try ScriptOptions.fromArgs(args) + catch + case _ => + sys.error(s"Wrong script parameters.\n${usageMessage}") + + val validationScript = scriptOptions.validationCommand.validationScript + val releases = Releases.fromRange(scriptOptions.releasesRange) + val releaseBisect = ReleaseBisect(validationScript, releases) + + releaseBisect.verifyEdgeReleases() + + if (!scriptOptions.dryRun) then + val (lastGoodRelease, firstBadRelease) = releaseBisect.bisectedGoodAndBadReleases() + println(s"Last good release: ${lastGoodRelease.version}") + println(s"First bad release: ${firstBadRelease.version}") + println("\nFinished bisecting releases\n") + + val commitBisect = CommitBisect(validationScript, bootstrapped = scriptOptions.bootstrapped, lastGoodRelease.hash, firstBadRelease.hash) + commitBisect.bisect() + + +case class ScriptOptions(validationCommand: ValidationCommand, dryRun: Boolean, bootstrapped: Boolean, releasesRange: ReleasesRange) +object ScriptOptions: + def fromArgs(args: Seq[String]) = + val defaultOptions = ScriptOptions( + validationCommand = null, + dryRun = false, + bootstrapped = false, + ReleasesRange(first = None, last = None) + ) + parseArgs(args, defaultOptions) + + private def parseArgs(args: Seq[String], options: ScriptOptions): ScriptOptions = + args match + case "--dry-run" :: argsRest => parseArgs(argsRest, options.copy(dryRun = true)) + case "--bootstrapped" :: argsRest => parseArgs(argsRest, options.copy(bootstrapped = true)) + case "--releases" :: argsRest => + val range = ReleasesRange.tryParse(argsRest.head).get + parseArgs(argsRest.tail, options.copy(releasesRange = range)) + case _ => + val command = ValidationCommand.fromArgs(args) + options.copy(validationCommand = command) + +enum ValidationCommand: + case Compile(args: Seq[String]) + case Run(args: Seq[String]) + case CustomValidationScript(scriptFile: File) + + def validationScript: File = this match + case Compile(args) => + ValidationScript.tmpScalaCliScript(command = "compile", args) + case Run(args) => + ValidationScript.tmpScalaCliScript(command = "run", args) + case CustomValidationScript(scriptFile) => + ValidationScript.copiedFrom(scriptFile) + +object ValidationCommand: + def fromArgs(args: Seq[String]) = args match + case Seq("compile", commandArgs*) => Compile(commandArgs) + case Seq("run", commandArgs*) => Run(commandArgs) + case Seq(path) => CustomValidationScript(new File(path)) + + +object ValidationScript: + def copiedFrom(file: File): File = + val fileContent = scala.io.Source.fromFile(file).mkString + tmpScript(fileContent) + + def tmpScalaCliScript(command: String, args: Seq[String]): File = tmpScript(s""" + |#!/usr/bin/env bash + |scala-cli ${command} -S "$$1" --server=false ${args.mkString(" ")} + |""".stripMargin + ) + + private def tmpScript(content: String): File = + val executableAttr = PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwxr-xr-x")) + val tmpPath = Files.createTempFile("scala-bisect-validator", "", executableAttr) + val tmpFile = tmpPath.toFile + + print(s"Bisecting with validation script: ${tmpPath.toAbsolutePath}\n") + print("#####################################\n") + print(s"${content}\n\n") + print("#####################################\n\n") + + tmpFile.deleteOnExit() + Files.write(tmpPath, content.getBytes(StandardCharsets.UTF_8)) + tmpFile + + +case class ReleasesRange(first: Option[String], last: Option[String]): + def filter(releases: Seq[Release]) = + def releaseIndex(version: String): Int = + val index = releases.indexWhere(_.version == version) + assert(index > 0, s"${version} matches no nightly compiler release") + index + + val startIdx = first.map(releaseIndex(_)).getOrElse(0) + val endIdx = last.map(releaseIndex(_) + 1).getOrElse(releases.length) + val filtered = releases.slice(startIdx, endIdx).toVector assert(filtered.nonEmpty, "No matching releases") filtered - -object VersionsRange: - def all = VersionsRange(None, None) - -object ParsedVersionsRange: - def unapply(range: String): Option[VersionsRange] = range match - case s"${first}...${last}" => Some(VersionsRange( +object ReleasesRange: + def all = ReleasesRange(None, None) + def tryParse(range: String): Option[ReleasesRange] = range match + case s"${first}...${last}" => Some(ReleasesRange( Some(first).filter(_.nonEmpty), Some(last).filter(_.nonEmpty) )) case _ => None -class ReleaseBisect(validationScriptPath: String): - def bisectedBadRelease(releases: Vector[Release]): Option[Release] = - Some(bisect(releases)) - .filter(!isGoodRelease(_)) +class Releases(val releases: Vector[Release]) + +object Releases: + lazy val allReleases: Vector[Release] = + val re = raw"""(?<=title=")(.+-bin-\d{8}-\w{7}-NIGHTLY)(?=/")""".r + val html = Source.fromURL("https://repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/") + re.findAllIn(html.mkString).map(Release.apply).toVector + + def fromRange(range: ReleasesRange): Vector[Release] = range.filter(allReleases) + +case class Release(version: String): + private val re = raw".+-bin-(\d{8})-(\w{7})-NIGHTLY".r + def date: String = + version match + case re(date, _) => date + case _ => sys.error(s"Could not extract date from release name: $version") + def hash: String = + version match + case re(_, hash) => hash + case _ => sys.error(s"Could not extract hash from release name: $version") - def bisect(releases: Vector[Release]): Release = - assert(releases.length > 1, "Need at least 2 releases to bisect") + override def toString: String = version + + +class ReleaseBisect(validationScript: File, allReleases: Vector[Release]): + assert(allReleases.length > 1, "Need at least 2 releases to bisect") + + private val isGoodReleaseCache = collection.mutable.Map.empty[Release, Boolean] + + def verifyEdgeReleases(): Unit = + println(s"Verifying the first release: ${allReleases.head.version}") + assert(isGoodRelease(allReleases.head), s"The evaluation script unexpectedly failed for the first checked release") + println(s"Verifying the last release: ${allReleases.last.version}") + assert(!isGoodRelease(allReleases.last), s"The evaluation script unexpectedly succeeded for the last checked release") + + def bisectedGoodAndBadReleases(): (Release, Release) = + val firstBadRelease = bisect(allReleases) + assert(!isGoodRelease(firstBadRelease), s"Bisection error: the 'first bad release' ${firstBadRelease.version} is not a bad release") + val lastGoodRelease = firstBadRelease.previous + assert(isGoodRelease(lastGoodRelease), s"Bisection error: the 'last good release' ${lastGoodRelease.version} is not a good release") + (lastGoodRelease, firstBadRelease) + + extension (release: Release) private def previous: Release = + val idx = allReleases.indexOf(release) + allReleases(idx - 1) + + private def bisect(releases: Vector[Release]): Release = if releases.length == 2 then if isGoodRelease(releases.head) then releases.last else releases.head @@ -109,48 +214,24 @@ class ReleaseBisect(validationScriptPath: String): else bisect(releases.take(releases.length / 2 + 1)) private def isGoodRelease(release: Release): Boolean = - println(s"Testing ${release.version}") - val result = Seq(validationScriptPath, release.version).! - val isGood = result == 0 - println(s"Test result: ${release.version} is a ${if isGood then "good" else "bad"} release\n") - isGood - -class Releases(val releases: Vector[Release]) - -object Releases: - private lazy val allReleases: Vector[String] = - val re = raw"""(?<=title=")(.+-bin-\d{8}-\w{7}-NIGHTLY)(?=/")""".r - val html = Source.fromURL("https://repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/") - re.findAllIn(html.mkString).toVector - - def fromRange(range: VersionsRange): Releases = Releases(range.filter(allReleases).map(Release(_))) - - case class Release(version: String): - private val re = raw".+-bin-(\d{8})-(\w{7})-NIGHTLY".r - def date: String = - version match - case re(date, _) => date - case _ => sys.error(s"Could not extract date from version $version") - def hash: String = - version match - case re(_, hash) => hash - case _ => sys.error(s"Could not extract hash from version $version") - - def previous(using r: Releases): Option[Release] = - val idx = r.releases.indexOf(this) - if idx == 0 then None - else Some(r.releases(idx - 1)) - - override def toString: String = version - -class CommitBisect(validationScriptPath: String): - def bisect(lastGoodHash: String, fistBadHash: String): Unit = + isGoodReleaseCache.getOrElseUpdate(release, { + println(s"Testing ${release.version}") + val result = Seq(validationScript.getAbsolutePath, release.version).! + val isGood = result == 0 + println(s"Test result: ${release.version} is a ${if isGood then "good" else "bad"} release\n") + isGood + }) + +class CommitBisect(validationScript: File, bootstrapped: Boolean, lastGoodHash: String, fistBadHash: String): + def bisect(): Unit = println(s"Starting bisecting commits $lastGoodHash..$fistBadHash\n") + val scala3CompilerProject = if bootstrapped then "scala3-compiler-bootstrapped" else "scala3-compiler" + val scala3Project = if bootstrapped then "scala3-bootstrapped" else "scala3" val bisectRunScript = s""" - |scalaVersion=$$(sbt "print scala3-compiler-bootstrapped/version" | tail -n1) + |scalaVersion=$$(sbt "print ${scala3CompilerProject}/version" | tail -n1) |rm -r out - |sbt "clean; scala3-bootstrapped/publishLocal" - |$validationScriptPath "$$scalaVersion" + |sbt "clean; ${scala3Project}/publishLocal" + |${validationScript.getAbsolutePath} "$$scalaVersion" """.stripMargin "git bisect start".! s"git bisect bad $fistBadHash".! diff --git a/project/scripts/examples/bisect-cli-example.sh b/project/scripts/examples/bisect-cli-example.sh index c1fe8141c623..6eb010cbe8bc 100755 --- a/project/scripts/examples/bisect-cli-example.sh +++ b/project/scripts/examples/bisect-cli-example.sh @@ -3,4 +3,4 @@ # Don't use this example script modified in place as it might disappear from the repo during a checkout. # Instead copy it to a different location first. -scala-cli compile -S "$1" file1.scala file2.scala +scala-cli compile -S "$1" --server=false file1.scala file2.scala diff --git a/project/scripts/examples/bisect-expect-example.exp b/project/scripts/examples/bisect-expect-example.exp index c921651a9333..4c094c373d30 100755 --- a/project/scripts/examples/bisect-expect-example.exp +++ b/project/scripts/examples/bisect-expect-example.exp @@ -6,7 +6,7 @@ set scalaVersion [lindex $argv 0] ;# Get the script argument set timeout 30 ;# Give scala-cli some time to download the compiler -spawn scala-cli repl -S "$scalaVersion" ;# Start the REPL +spawn scala-cli repl -S "$scalaVersion" --server=false ;# Start the REPL expect "scala>" ;# REPL has started set timeout 5 send -- "Seq.empty.len\t" ;# Tab pressed to trigger code completion