New ci

build.number

@@ -1,9 +1,9 @@
 #Tue Sep 11 19:21:09 CEST 2007
 version.major=2
 version.minor=11
-version.patch=5
+version.patch=6
 # This is the -N part of a version.  if it's 0, it's dropped from maven versions.
 version.bnum=0
 
-# Note: To build a release run ant with -Dbuild.release=true
-# To build an RC, run ant with -Dmaven.version.suffix=-RCN
+# To build a release, see scripts/jobs/scala-release-2.11.x-build
+# (normally run by the eponymous job on scala-ci.typesafe.com).

scripts/common

@@ -0,0 +1,153 @@
+# This is for forcibly stopping the job from a subshell (see test
+# below).
+trap "exit 1" TERM
+export TOP_PID=$$
+set -e
+
+# Known problems : does not fare well with interrupted, partial
+# compilations. We should perhaps have a multi-dependency version
+# of do_i_have below
+
+LOGGINGDIR="$WORKSPACE/logs"
+mkdir -p $LOGGINGDIR
+
+unset SBT_HOME
+SBT_HOME="$WORKSPACE/.sbt"
+mkdir -p $SBT_HOME
+IVY_CACHE="$WORKSPACE/.ivy2"
+mkdir -p $IVY_CACHE
+rm -rf $IVY_CACHE/cache/org.scala-lang
+
+# temp dir where all 'non-build' operation are performed
+TMP_ROOT_DIR=$(mktemp -d -t pr-scala.XXXX)
+TMP_DIR="${TMP_ROOT_DIR}/tmp"
+mkdir "${TMP_DIR}"
+
+
+# detect sed version and how to enable extended regexes
+SEDARGS="-n$(if (echo "a" | sed -nE "s/a/b/" &> /dev/null); then echo E; else echo r; fi)"
+
+
+
+# :docstring test:
+# Usage: test <argument ..>
+# Executes <argument ..>, logging the launch of the command to the
+# main log file, and kills global script execution with the TERM
+# signal if the commands ends up failing.
+# DO NOT USE ON FUNCTIONS THAT DECLARE VARIABLES,
+# AS YOU'LL BE RUNNING IN A SUBSHELL AND VARIABLE DECLARATIONS WILL BE LOST
+# :end docstring:
+
+function test() {
+    echo "### $@"
+    "$@"
+    status=$?
+    if [ $status -ne 0 ]; then
+        say "### ERROR with $1"
+        kill -s TERM $TOP_PID
+    fi
+}
+
+# :docstring say:
+# Usage: say <argument ..>
+# Prints <argument ..> to both console and the main log file.
+# :end docstring:
+
+function say(){
+    (echo "$@") | tee -a $LOGGINGDIR/compilation-$SCALADATE-$SCALAHASH.log
+}
+
+# General debug logging
+# $* - message
+function debug () {
+  echo "----- $*"
+}
+
+function parseScalaProperties(){
+  propFile="$baseDir/$1"
+  if [ ! -f $propFile ]; then
+    echo "Property file $propFile not found."
+    exit 1
+  else
+    awk -f "$scriptsDir/readproperties.awk" "$propFile" > "$propFile.sh"
+    . "$propFile.sh" # yeah yeah, not that secure, improvements welcome (I tried, but bash made me cry again)
+  fi
+}
+
+
+## TAKEN FROM UBER-BUILD, except that it "returns" (via $RES) true/false
+# Check if an artifact is available
+# $1 - groupId
+# $2 - artifacId
+# $3 - version
+# $4 - extra repository to look in (optional)
+# return value in $RES
+function checkAvailability () {
+  pushd "${TMP_DIR}"
+  rm -rf *
+
+# pom file for the test project
+  cat > pom.xml << EOF
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>com.typesafe</groupId>
+  <artifactId>typesafeDummy</artifactId>
+  <packaging>war</packaging>
+  <version>1.0-SNAPSHOT</version>
+  <name>Dummy</name>
+  <url>http://127.0.0.1</url>
+  <dependencies>
+    <dependency>
+      <groupId>$1</groupId>
+      <artifactId>$2</artifactId>
+      <version>$3</version>
+    </dependency>
+  </dependencies>
+  <repositories>
+    <repository>
+      <id>sonatype.snapshot</id>
+      <name>Sonatype maven snapshot repository</name>
+      <url>https://oss.sonatype.org/content/repositories/snapshots</url>
+      <snapshots>
+        <updatePolicy>daily</updatePolicy>
+      </snapshots>
+    </repository>
+EOF
+
+  if [ -n "$4" ]
+  then
+# adds the extra repository
+    cat >> pom.xml << EOF
+    <repository>
+      <id>extrarepo</id>
+      <name>extra repository</name>
+      <url>$4</url>
+    </repository>
+EOF
+  fi
+
+  cat >> pom.xml << EOF
+  </repositories>
+</project>
+EOF
+
+  set +e
+  mvn "${MAVEN_ARGS[@]}" compile &> "${TMP_DIR}/mvn.log"
+  RES=$?
+  # Quiet the maven, but allow diagnosing problems.
+  grep -i downloading "${TMP_DIR}/mvn.log"
+  grep -i exception "${TMP_DIR}/mvn.log"
+  grep -i error "${TMP_DIR}/mvn.log"
+  set -e
+
+# log the result
+  if [ ${RES} == 0 ]
+  then
+    debug "$1:$2:jar:$3 found !"
+    RES=true
+  else
+    debug "$1:$2:jar:$3 not found !"
+    RES=false
+  fi
+  popd
+}

scripts/jobs/scala-release-2.11.x-build → scripts/jobs/integrate/bootstrap

@@ -1,4 +1,9 @@
 #!/bin/bash -e
+# TODO: different scripts for the different phases -- usually we don't need to bootstrap the modules,
+# since we can use the previous version of scala for STARR as well as for compiling the modules (assuming it's binary compatible)
+# We should move away from the complicated bootstrap and set up our release schedule so we always have a previous build that satisfies these criteria.
+# (Potentially trivially, by splitting up this script, and publishing locker as if it were a real release.)
+
 # requirements:
 # sbtCmd must point to sbt from sbt-extras (this is the standard on the Scala jenkins, so we only support that one)
 # - ~/.sonatype-curl that consists of user = USER:PASS
@@ -66,7 +71,6 @@
 # set to something besides the default to build nightly snapshots of the modules instead of some tagged version
 moduleVersioning=${moduleVersioning-"versions.properties"}
 
-baseDir=${WORKSPACE-`pwd`}
 publishPrivateTask=${publishPrivateTask-"publish"}
 publishSonatypeTaskCore=${publishSonatypeTaskCore-"publish-signed"}
 publishSonatypeTaskModules=${publishSonatypeTaskModules-"publish-signed"}
@@ -88,67 +92,11 @@ forceRebuild=${forceRebuild-no}
 antBuildTask="${antBuildTask-nightly}" # TESTING leave empty to avoid the sanity check (don't set it to "init" because ant will croak)
 clean="clean" # TESTING leave empty to speed up testing
 
-scriptsDir="$WORKSPACE/scripts"
-
-# This is for forcibly stopping the job from a subshell (see test
-# below).
-trap "exit 1" TERM
-export TOP_PID=$$
-set -e
-
-# Known problems : does not fare well with interrupted, partial
-# compilations. We should perhaps have a multi-dependency version
-# of do_i_have below
-
-LOGGINGDIR="$WORKSPACE/logs"
-mkdir -p $LOGGINGDIR
-
-unset SBT_HOME
-SBT_HOME="$WORKSPACE/.sbt"
-mkdir -p $SBT_HOME
-IVY_CACHE="$WORKSPACE/.ivy2"
-mkdir -p $IVY_CACHE
-rm -rf $IVY_CACHE/cache/org.scala-lang
-
-# temp dir where all 'non-build' operation are performed
-TMP_ROOT_DIR=$(mktemp -d -t pr-scala.XXXX)
-TMP_DIR="${TMP_ROOT_DIR}/tmp"
-mkdir "${TMP_DIR}"
-
-
-# detect sed version and how to enable extended regexes
-SEDARGS="-n$(if (echo "a" | sed -nE "s/a/b/" &> /dev/null); then echo E; else echo r; fi)"
-
-
 
-# :docstring test:
-# Usage: test <argument ..>
-# Executes <argument ..>, logging the launch of the command to the
-# main log file, and kills global script execution with the TERM
-# signal if the commands ends up failing.
-# DO NOT USE ON FUNCTIONS THAT DECLARE VARIABLES,
-# AS YOU'LL BE RUNNING IN A SUBSHELL AND VARIABLE DECLARATIONS WILL BE LOST
-# :end docstring:
-
-function test() {
-    echo "### $@"
-    "$@"
-    status=$?
-    if [ $status -ne 0 ]; then
-        say "### ERROR with $1"
-        kill -s TERM $TOP_PID
-    fi
-}
-
-# :docstring say:
-# Usage: say <argument ..>
-# Prints <argument ..> to both console and the main log file.
-# :end docstring:
-
-function say(){
-    (echo "$@") | tee -a $LOGGINGDIR/compilation-$SCALADATE-$SCALAHASH.log
-}
 
+baseDir=${WORKSPACE-`pwd`}
+scriptsDir="$baseDir/scripts"
+. $scriptsDir/common
 
 # we must change ivy home to get a fresh ivy cache, otherwise we get half-bootstrapped scala
 # rm it in case it existed (and there's no ivy2-shadow, which indicates we're running in a TESTING environment)...
@@ -163,17 +111,6 @@ mkdir -p $baseDir/resolutionScratch_
 privateCred="private-repo"
 privateRepo="http://private-repo.typesafe.com/typesafe/scala-release-temp/"
 
-function parseScalaProperties(){
-  propFile="$baseDir/$1"
-  if [ ! -f $propFile ]; then
-    echo "Property file $propFile not found."
-    exit 1
-  else
-    awk -f "$scriptsDir/readproperties.awk" "$propFile" > "$propFile.sh"
-    . "$propFile.sh" # yeah yeah, not that secure, improvements welcome (I tried, but bash made me cry again)
-  fi
-}
-
 ##### git
 gfxd() {
   git clean -fxd # TESTING

scripts/jobs/validate/publish-core

@@ -0,0 +1,44 @@
+#!/bin/bash -e
+# This script publishes the core of Scala to maven for use as locker downstream,
+# and saves the relevant properties used in its build artifacts, versions.properties.
+# (This means we'll use locker instead of quick downstream in dbuild.
+#  The only downside is that backend improvements don't improve compiler performance itself until they are in STARR).
+# The version is suffixed with "-${sha:0:7}-SNAPSHOT"
+
+baseDir=${WORKSPACE-`pwd`}
+scriptsDir="$baseDir/scripts"
+. $scriptsDir/common
+
+case $prDryRun in
+  yep)
+    echo "DRY RUN"
+    mkdir -p build/pack ; mkdir -p dists/maven/latest
+    ;;
+  *)
+    sha=$(git rev-parse HEAD) # TODO: warn if $repo_ref != $sha (we shouldn't do PR validation using symbolic gitrefs)
+    echo "sha/repo_ref == $sha/$repo_ref ?"
+
+    parseScalaProperties build.number
+
+    ./pull-binary-libs.sh
+    # "noyoudont" is there juuuust in case
+    antDeployArgs="-Dmaven.version.suffix=\"-${sha:0:7}-SNAPSHOT\" -Dremote.snapshot.repository=$prRepoUrl -Drepository.credentials.id=pr-scala -Dremote.release.repository=noyoudont"
+
+    echo ">>> Getting Scala version number."
+    ant -q $antDeployArgs init
+    parseScalaProperties buildcharacter.properties # produce maven_version_number
+
+    echo ">>> Checking availability of Scala ${maven_version_number} in $prRepoUrl."
+    checkAvailability "org.scala-lang" "scala-library"  "${maven_version_number}" $prRepoUrl; libraryAvailable=$RES
+    checkAvailability "org.scala-lang" "scala-reflect"  "${maven_version_number}" $prRepoUrl; reflectAvailable=$RES
+    checkAvailability "org.scala-lang" "scala-compiler" "${maven_version_number}" $prRepoUrl; compilerAvailable=$RES
+
+    if $libraryAvailable && $reflectAvailable && $compilerAvailable; then
+      echo "Scala core already built!"
+    else
+      ant $antDeployArgs $antBuildArgs publish-opt-nodocs
+    fi
+
+    mv buildcharacter.properties jenkins.properties # parsed by the jenkins job
+    ;;
+esac

scripts/jobs/validate/test

@@ -0,0 +1,17 @@
+#!/bin/bash -e
+
+case $prDryRun in
+  yep)
+    echo "DRY RUN"
+    ;;
+  *)
+    ./pull-binary-libs.sh
+
+    # build quick using STARR built upstream, as specified by maven_version_number
+    # (in that sense it's locker, since it was built with starr by that upstream job)
+    ant -Dstarr.version=$maven_version_number \
+       -Dscalac.args.optimise=-optimise \
+       -Dlocker.skip=1 -Dstarr.use.released=1 -Dextra.repo.url=$prRepoUrl \
+       $testExtraArgs ${testTarget-test.core docs.done}
+    ;;
+esac