Switch Windows CI build to sbt (w/ some sbt build improvements)

- Use sbt in `integrate/windows`: This essentially combines the Unix CI
  jobs `validate/publish-core` and `validate/test`, first publishing a
  local release built with STARR and then building a new version with
  that and running all tests on it.

- Unify repository handling across build scripts: A new function
  `generateRepositoriesConfig` in `common`, based on the existing code
  in `integrate/bootstrap`, writes the `repositories` file for sbt,
  either with or without an extra bootstrap repository for resolving a
  previously built version. It is used in all CI scripts to ensure that
  artifacts are only resolved through the sanctioned proxies and
  upstream repositories.

- The repository URL arguments in `setupPublishCore` and
  `setupValidateTest` are now optional as well. These commands are used
  without a URL from `integrate/windows`, which publishes to `local`
  instead of a temporary remote repository.

- `testAll` is now a task instead of a command. It runs the same
  sequence of sub-tasks as before but does not propagate failures
  immediately. It always runs all subtasks and reports errors at the
  end.

- The `generateBuildCharacterPropertiesFile` task now includes all
  properties from `versions.properties` (whose values have potentially
  been overwritten with `-D` options) in `buildcharacter.properties`.

Author: szeiger

build.sbt

@@ -769,6 +769,32 @@ lazy val root: Project = (project in file("."))
       GenerateAnyVals.run(dir.getAbsoluteFile)
       state
     },
+    testAll := {
+      val results = ScriptCommands.sequence[Result[Unit]](List(
+        (Keys.test in Test in junit).result,
+        (testOnly in IntegrationTest in testP).toTask(" -- run pos neg jvm").result,
+        (testOnly in IntegrationTest in testP).toTask(" -- res scalap specialized scalacheck").result,
+        (testOnly in IntegrationTest in testP).toTask(" -- instrumented presentation").result,
+        (testOnly in IntegrationTest in testP).toTask(" -- --srcpath scaladoc").result,
+        (Keys.test in Test in osgiTestFelix).result,
+        (Keys.test in Test in osgiTestEclipse).result,
+        (MiMa.mima in library).result,
+        (MiMa.mima in reflect).result,
+        Def.task(()).dependsOn( // Run these in parallel:
+          doc in Compile in library,
+          doc in Compile in reflect,
+          doc in Compile in compiler,
+          doc in Compile in scalap
+        ).result
+      )).value
+      val failed = results.map(_.toEither).collect { case Left(i) => i }
+      if(failed.nonEmpty) {
+        val log = streams.value.log
+        log.error(s"${failed.size} of ${results.length} test tasks failed:")
+        failed.foreach(i => log.error(s"  - $i"))
+        throw new RuntimeException
+      }
+    },
     antStyle := false,
     incOptions := incOptions.value.withNameHashing(!antStyle.value).withAntStyle(antStyle.value)
   )
@@ -838,6 +864,7 @@ lazy val buildDirectory = settingKey[File]("The directory where all build produc
 lazy val mkBin = taskKey[Seq[File]]("Generate shell script (bash or Windows batch).")
 lazy val mkQuick = taskKey[File]("Generate a full build, including scripts, in build/quick")
 lazy val mkPack = taskKey[File]("Generate a full build, including scripts, in build/pack")
+lazy val testAll = taskKey[Unit]("Run all test tasks sequentially")
 
 // Defining these settings is somewhat redundant as we also redefine settings that depend on them.
 // However, IntelliJ's project import works better when these are set correctly.

project/ScriptCommands.scala

@@ -7,24 +7,29 @@ object ScriptCommands {
   def all = Seq(
     setupPublishCore,
     setupValidateTest,
-    setupBootstrapStarr, setupBootstrapLocker, setupBootstrapQuick, setupBootstrapPublish,
-    testAll
+    setupBootstrapStarr, setupBootstrapLocker, setupBootstrapQuick, setupBootstrapPublish
   )
 
-  /** Set up the environment for `validate/publish-core`. The argument is the Artifactory snapshot repository URL. */
-  def setupPublishCore = setup("setupPublishCore") { case Seq(url) =>
+  /** Set up the environment for `validate/publish-core`.
+    * The optional argument is the Artifactory snapshot repository URL. */
+  def setupPublishCore = setup("setupPublishCore") { args =>
     Seq(
       baseVersionSuffix in Global := "SHA-SNAPSHOT"
-    ) ++ publishTarget(url) ++ noDocs ++ enableOptimizer
+    ) ++ (args match {
+      case Seq(url) => publishTarget(url)
+      case Nil => Nil
+    }) ++ noDocs ++ enableOptimizer
   }
 
-  /** Set up the environment for `validate/test`. The argument is the Artifactory snapshot repository URL. */
-  def setupValidateTest = setup("setupValidateTest") { case Seq(url) =>
-    //TODO When ant is gone, pass starr version as an argument to this command instead of using version.properties
+  /** Set up the environment for `validate/test`.
+    * The optional argument is the Artifactory snapshot repository URL. */
+  def setupValidateTest = setup("setupValidateTest") { args =>
     Seq(
-      resolvers in Global += "scala-pr" at url,
       testOptions in IntegrationTest in LocalProject("test") ++= Seq(Tests.Argument("--show-log"), Tests.Argument("--show-diff"))
-    ) ++ enableOptimizer
+    ) ++ (args match {
+      case Seq(url) => Seq(resolvers in Global += "scala-pr" at url)
+      case Nil => Nil
+    }) ++ enableOptimizer
   }
 
   /** Set up the environment for building STARR in `validate/bootstrap`. The arguments are:
@@ -63,9 +68,7 @@ object ScriptCommands {
   /** Set up the environment for publishing in `validate/bootstrap`. The arguments are:
     * - Temporary bootstrap repository URL for resolving modules
     * - Version number to publish
-    * All artifacts are published to Sonatype. GPG signing has to be configured from the
-    * shell script after `setupBootstrapPublish` because we don't pull the GPG plugin in
-    * by default, so we cannot reference its keys statically. */
+    * All artifacts are published to Sonatype. */
   def setupBootstrapPublish = setup("setupBootstrapPublish") { case Seq(url, ver) =>
     // Define a copy of the setting key here in case the plugin is not part of the build
     val pgpPassphrase = SettingKey[Option[Array[Char]]]("pgp-passphrase", "The passphrase associated with the secret used to sign artifacts.", KeyRanks.BSetting)
@@ -95,25 +98,18 @@ object ScriptCommands {
     publishArtifact in (Compile, packageDoc) in ThisBuild := false
   )
 
-  private[this] def publishTarget(url: String) = Seq(
+  private[this] def publishTarget(url: String) = {
     // Append build.timestamp to Artifactory URL to get consistent build numbers (see https://github.com/sbt/sbt/issues/2088):
-    publishTo in Global := Some("scala-pr-publish" at url.replaceAll("/$", "") + ";build.timestamp=" + System.currentTimeMillis)
-  )
+    val url2 = if(url.startsWith("file:")) url else url.replaceAll("/$", "") + ";build.timestamp=" + System.currentTimeMillis
+    Seq(publishTo in Global := Some("scala-pr-publish" at url2))
+  }
 
-  def testAll = Command.command("testAll") { state =>
-    val cmds = Seq(
-      "test",
-      "partest run pos neg jvm",
-      "partest res scalap specialized scalacheck",
-      "partest instrumented presentation",
-      "partest --srcpath scaladoc",
-      "osgiTestFelix/test",
-      "osgiTestEclipse/test",
-      "library/mima",
-      "reflect/mima",
-      "doc"
-    )
-    state.log.info(cmds.mkString("Running all tests: \"", "\", \"", "\""))
-    cmds ::: state
+  /** Like `Def.sequential` but accumulate all results */
+  def sequence[B](tasks: List[Def.Initialize[Task[B]]]): Def.Initialize[Task[List[B]]] = tasks match {
+    case Nil => Def.task { Nil }
+    case x :: xs => Def.taskDyn {
+      val v = x.value
+      sequence(xs).apply((t: Task[List[B]]) => t.map(l => v :: l))
+    }
   }
 }

project/VersionUtil.scala

@@ -94,7 +94,7 @@ object VersionUtil {
   }
 
   private lazy val generateBuildCharacterPropertiesFileImpl: Def.Initialize[Task[File]] = Def.task {
-    writeProps(versionProperties.value.toMap, (baseDirectory in ThisBuild).value / "buildcharacter.properties")
+    writeProps(versionProperties.value.toMap ++ versionProps, (baseDirectory in ThisBuild).value / "buildcharacter.properties")
   }
 
   private def writeProps(m: Map[String, String], propFile: File): File = {

scripts/common

@@ -154,3 +154,23 @@ EOF
   fi
   popd
 }
+
+# Generate a repositories file with all allowed repositories in our build environment.
+# Takes one optional argument, the private repository URL.
+# See http://www.scala-sbt.org/0.13/docs/Proxy-Repositories.html
+function generateRepositoriesConfig() {
+  jcenterCacheUrl=${jcenterCacheUrl-"https://scala-ci.typesafe.com/artifactory/jcenter/"}
+  sbtRepositoryConfig="$scriptsDir/sbt-repositories-config"
+  echo > "$sbtRepositoryConfig" '[repositories]'
+  if [ -n "$1" ]
+  then
+    echo >> "$sbtRepositoryConfig" "  private-repo: $1"
+  fi
+  cat >> "$sbtRepositoryConfig" << EOF
+  jcenter-cache: $jcenterCacheUrl
+  typesafe-ivy-releases: https://dl.bintray.com/typesafe/ivy-releases/, [organisation]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext], bootOnly
+  sbt-plugin-releases: https://dl.bintray.com/sbt/sbt-plugin-releases/, [organisation]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext]
+  maven-central
+  local
+EOF
+}

scripts/jobs/integrate/bootstrap

@@ -85,10 +85,8 @@
 moduleVersioning=${moduleVersioning-"versions.properties"}
 
 publishPrivateTask=${publishPrivateTask-"publish"}
-publishSonatypeTaskCore=${publishSonatypeTaskCore-"publish-signed"}
-publishSonatypeTaskModules=${publishSonatypeTaskModules-"publish-signed"}
-publishStarrPrivateTask=${publishStarrPrivateTask-$publishPrivateTask} # set to "init" to speed up testing of the script (if you already built STARR before)
-publishLockerPrivateTask=${publishLockerPrivateTask-$publishPrivateTask} # set to "init" to speed up testing of the script (if you already built locker before)
+publishSonatypeTaskCore=${publishSonatypeTaskCore-"publishSigned"}
+publishSonatypeTaskModules=${publishSonatypeTaskModules-"publishSigned"}
 
 forceRebuild=${forceRebuild-no}
 
@@ -111,21 +109,7 @@ mkdir -p $baseDir/resolutionScratch_
 # repo used to publish "locker" scala to (to start the bootstrap)
 releaseTempRepoCred="private-repo"
 releaseTempRepoUrl=${releaseTempRepoUrl-"https://scala-ci.typesafe.com/artifactory/scala-release-temp/"}
-jcenterCacheUrl=${jcenterCacheUrl-"https://scala-ci.typesafe.com/artifactory/jcenter/"}
-
-# Used below in sbtArgs since we use a dedicated repository to share artifcacts between jobs,
-# so we need to configure SBT to use these rather than its default, Maven Central.
-# See http://www.scala-sbt.org/0.13/docs/Proxy-Repositories.html
-sbtRepositoryConfig="$scriptsDir/repositories-scala-release"
-cat > "$sbtRepositoryConfig" << EOF
-[repositories]
-  private-repo: $releaseTempRepoUrl
-  jcenter-cache: $jcenterCacheUrl
-  typesafe-ivy-releases: https://dl.bintray.com/typesafe/ivy-releases/, [organisation]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext], bootOnly
-  sbt-plugin-releases: https://dl.bintray.com/sbt/sbt-plugin-releases/, [organisation]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext]
-  maven-central
-  local
-EOF
+generateRepositoriesConfig $releaseTempRepoUrl
 
 ##### git
 gfxd() {
@@ -575,7 +559,7 @@ publishSonatype() {
       -Dstarr.version=$SCALA_VER \
       ${updatedModuleVersions[@]} \
       "setupBootstrapPublish $releaseTempRepoUrl $SCALA_VER" \
-      publishSigned | grep -v "was too long to be displayed in the webview, and will be left out"
+      $publishSonatypeTaskCore | grep -v "was too long to be displayed in the webview, and will be left out"
 
   echo "### Publishing modules to sonatype"
   # build/test/publish scala core modules to sonatype (this will start a new staging repo)

scripts/jobs/integrate/windows

@@ -1,15 +1,19 @@
 #!/bin/bash
 
-./pull-binary-libs.sh
-
-export ANT_OPTS="-Dfile.encoding=UTF-8 -server -XX:+AggressiveOpts -XX:+UseParNewGC -Xmx2G -Xss1M -XX:MaxPermSize=512M -XX:ReservedCodeCacheSize=128M"
-
-# TODO: don't hardcode these paths -- should be in scala/scala-jenkins-infra, passed in through env vars from jenkins
-export PATH='/cygdrive/c/Program Files/Java/jdk1.8.0_92/bin:/cygdrive/c/apache-ant-1.9.6/bin:/cygdrive/c/Program Files (x86)/Git-2.5.3/Cmd:/bin:/usr/bin:'
-export JAVA_HOME='C:/Program Files/Java/jdk1.8.0_92'
+baseDir=${WORKSPACE-`pwd`}
+scriptsDir="$baseDir/scripts"
+. $scriptsDir/common
 
 java -version
 javac -version
-ant -version
 
-ant test-opt
+generateRepositoriesConfig
+
+SBT="java $JAVA_OPTS -Dsbt.ivy.home=$WORKSPACE/.ivy2 -jar $sbtLauncher -Dsbt.override.build.repos=true -Dsbt.repository.config=$sbtRepositoryConfig"
+
+# Build locker with STARR
+$SBT --warn "setupPublishCore" generateBuildCharacterPropertiesFile publishLocal
+
+# Build quick and run the tests
+parseScalaProperties buildcharacter.properties
+$SBT -Dstarr.version=$maven_version_number --warn "setupValidateTest" testAll | grep -v "was too long to be displayed in the webview, and will be left out"

scripts/jobs/validate/publish-core

@@ -9,14 +9,17 @@ baseDir=${WORKSPACE-`pwd`}
 scriptsDir="$baseDir/scripts"
 . $scriptsDir/common
 
+generateRepositoriesConfig $prRepoUrl
+SBT="$SBT_CMD -Dsbt.override.build.repos=true -Dsbt.repository.config=$sbtRepositoryConfig"
+
 case $prDryRun in
   yep)
     echo "DRY RUN"
     mkdir -p build/pack ; mkdir -p dists/maven/latest
     ;;
   *)
     echo ">>> Getting Scala version number."
-    $SBT_CMD --warn "setupPublishCore $prRepoUrl" generateBuildCharacterPropertiesFile
+    $SBT --warn "setupPublishCore $prRepoUrl" generateBuildCharacterPropertiesFile
     parseScalaProperties buildcharacter.properties # produce maven_version_number
 
     echo ">>> Checking availability of Scala ${maven_version_number} in $prRepoUrl."
@@ -27,7 +30,7 @@ case $prDryRun in
     if $libraryAvailable && $reflectAvailable && $compilerAvailable; then
       echo "Scala core already built!"
     else
-      $SBT_CMD --warn "setupPublishCore $prRepoUrl" publish
+      $SBT --warn "setupPublishCore $prRepoUrl" publish
     fi
 
     mv buildcharacter.properties jenkins.properties # parsed by the jenkins job

scripts/jobs/validate/test

@@ -4,6 +4,9 @@ baseDir=${WORKSPACE-`pwd`}
 scriptsDir="$baseDir/scripts"
 . $scriptsDir/common
 
+generateRepositoriesConfig $prRepoUrl
+SBT="$SBT_CMD -Dsbt.override.build.repos=true -Dsbt.repository.config=$sbtRepositoryConfig"
+
 case $prDryRun in
 
   yep)
@@ -15,12 +18,12 @@ case $prDryRun in
     # build quick using STARR built upstream, as specified by scalaVersion
     # (in that sense it's locker, since it was built with starr by that upstream job);
     # and run JUnit tests, partest, OSGi tests, MiMa and scaladoc
-    $SBT_CMD \
+    $SBT \
        -Dstarr.version=$scalaVersion \
        --warn \
        "setupValidateTest $prRepoUrl" \
        $testExtraArgs \
-       testAll
+       testAll | grep -v "was too long to be displayed in the webview, and will be left out"
 
     ;;