Add --update-checkfiles to testCompilation

Author: nicolasstucki

compiler/test/dotc/comptest.scala

@@ -11,6 +11,7 @@ object comptest extends ParallelTesting {
   def safeMode = false
   def isInteractive = true
   def testFilter = None
+  def updateCheckFiles: Boolean = false
 
   val posDir = "./tests/pos/"
   val negDir = "./tests/neg/"

compiler/test/dotty/Properties.scala

@@ -20,6 +20,9 @@ object Properties {
    */
   val testsFilter: Option[String] = sys.props.get("dotty.tests.filter")
 
+  /** Tests should override the checkfiles with the current output */
+  val testsUpdateCheckfile: Boolean = propIsNullOrTrue("dotty.tests.updateCheckfiles")
+
   /** When set, the run tests are only compiled - not run, a warning will be
    *  issued
    */

compiler/test/dotty/tools/dotc/BootstrappedOnlyCompilationTests.scala

@@ -7,13 +7,8 @@ import org.junit.Assert._
 import org.junit.Assume._
 import org.junit.experimental.categories.Category
 
-import java.nio.file._
-import java.util.stream.{ Stream => JStream }
-import scala.collection.JavaConverters._
-import scala.util.matching.Regex
 import scala.concurrent.duration._
 import vulpix._
-import dotty.tools.io.JFile
 
 @Category(Array(classOf[BootstrappedOnlyTests]))
 class BootstrappedOnlyCompilationTests extends ParallelTesting {
@@ -28,6 +23,7 @@ class BootstrappedOnlyCompilationTests extends ParallelTesting {
   def safeMode = Properties.testsSafeMode
   def isInteractive = SummaryReport.isInteractive
   def testFilter = Properties.testsFilter
+  def updateCheckFiles: Boolean = Properties.testsUpdateCheckfile
 
   // Positive tests ------------------------------------------------------------
 

compiler/test/dotty/tools/dotc/CompilationTests.scala

@@ -27,6 +27,7 @@ class CompilationTests extends ParallelTesting {
   def safeMode = Properties.testsSafeMode
   def isInteractive = SummaryReport.isInteractive
   def testFilter = Properties.testsFilter
+  def updateCheckFiles: Boolean = Properties.testsUpdateCheckfile
 
   // Positive tests ------------------------------------------------------------
 
@@ -303,7 +304,7 @@ class CompilationTests extends ParallelTesting {
 }
 
 object CompilationTests {
-  implicit val summaryReport: SummaryReporting = new SummaryReport(updateCheckFiles = false)
+  implicit val summaryReport: SummaryReporting = new SummaryReport
   @AfterClass def cleanup(): Unit = summaryReport.echoSummary()
 
   def sources(paths: JStream[Path], excludedFiles: List[String] = Nil): List[String] = {

compiler/test/dotty/tools/dotc/FromTastyTests.scala

@@ -20,7 +20,7 @@ class FromTastyTests extends ParallelTesting {
   def safeMode = Properties.testsSafeMode
   def isInteractive = SummaryReport.isInteractive
   def testFilter = Properties.testsFilter
-
+  def updateCheckFiles: Boolean = Properties.testsUpdateCheckfile
 
   @Test def posTestFromTasty: Unit = {
     // Can be reproduced with
@@ -51,6 +51,6 @@ class FromTastyTests extends ParallelTesting {
 }
 
 object FromTastyTests {
-  implicit val summaryReport: SummaryReporting = new SummaryReport(updateCheckFiles = false)
+  implicit val summaryReport: SummaryReporting = new SummaryReport
   @AfterClass def cleanup(): Unit = summaryReport.echoSummary()
 }

compiler/test/dotty/tools/dotc/IdempotencyTests.scala

@@ -24,6 +24,7 @@ class IdempotencyTests extends ParallelTesting {
   def safeMode = Properties.testsSafeMode
   def isInteractive = SummaryReport.isInteractive
   def testFilter = Properties.testsFilter
+  def updateCheckFiles: Boolean = Properties.testsUpdateCheckfile
 
   @Category(Array(classOf[SlowTests]))
   @Test def idempotency: Unit = {
@@ -72,6 +73,6 @@ class IdempotencyTests extends ParallelTesting {
 }
 
 object IdempotencyTests {
-  implicit val summaryReport: SummaryReporting = new SummaryReport(updateCheckFiles = false)
+  implicit val summaryReport: SummaryReporting = new SummaryReport
   @AfterClass def cleanup(): Unit = summaryReport.echoSummary()
 }

compiler/test/dotty/tools/vulpix/ParallelTesting.scala

@@ -46,6 +46,9 @@ trait ParallelTesting extends RunnerOrchestration { self =>
    */
   def testFilter: Option[String]
 
+  /** Tests should override the checkfiles with the current output */
+  def updateCheckFiles: Boolean
+
   /** A test source whose files or directory of files is to be compiled
    *  in a specific way defined by the `Test`
    */
@@ -550,7 +553,7 @@ trait ParallelTesting extends RunnerOrchestration { self =>
 
                     if (output.mkString(EOL) != check) {
                       val outFile = dotty.tools.io.File(checkFile.toPath).addExtension(".out")
-                      if (summaryReport.updateCheckFiles) {
+                      if (updateCheckFiles) {
                         updateCheckFile(checkFile, output)
                       } else {
                         outFile.writeAll(output.mkString("", EOL, ""))
@@ -642,7 +645,7 @@ trait ParallelTesting extends RunnerOrchestration { self =>
             // Fail target:
             failTestSource(testSource)
 
-            if (summaryReport.updateCheckFiles)
+            if (updateCheckFiles)
               updateCheckFile(checkFile.get, outputLines)
           }
         }
@@ -781,7 +784,7 @@ trait ParallelTesting extends RunnerOrchestration { self =>
           val expexted = Source.fromFile(checkFile, "UTF-8").getLines().toList
           for (msg <- diffMessage(sourceName, actual, expexted)) {
             fail(msg)
-            if (summaryReport.updateCheckFiles)
+            if (updateCheckFiles)
               updateCheckFile(checkFile, actual)
           }
         }

compiler/test/dotty/tools/vulpix/SummaryReport.scala

@@ -41,8 +41,6 @@ trait SummaryReporting {
   /** Echoes contents of `it` to file *immediately* then flushes */
   def echoToLog(it: Iterator[String]): Unit
 
-  /** Echoes outputs of the test into the thier checkfiles */
-  def updateCheckFiles: Boolean
 }
 
 /** A summary report that doesn't do anything */
@@ -62,7 +60,7 @@ final class NoSummaryReport extends SummaryReporting {
 /** A summary report that logs to both stdout and the `TestReporter.logWriter`
  *  which outputs to a log file in `./testlogs/`
  */
-final class SummaryReport(val updateCheckFiles: Boolean) extends SummaryReporting {
+final class SummaryReport extends SummaryReporting {
   import scala.collection.JavaConverters._
 
   private val startingMessages = new java.util.concurrent.ConcurrentLinkedDeque[String]

compiler/test/dotty/tools/vulpix/VulpixMetaTests.scala

@@ -4,7 +4,6 @@ package vulpix
 import org.junit.Test
 import org.junit.experimental.categories.Category
 import scala.concurrent.duration._
-import dotty.Properties
 import TestConfiguration._
 
 /** Meta tests for the Vulpix test suite. This test follows the structure of
@@ -19,8 +18,9 @@ class VulpixMetaTests extends ParallelTesting {
   def safeMode = false // Don't fork a new VM after each run test
   def isInteractive = false // Don't beautify output for interactive use.
   def testFilter = None // Run all the tests.
+  def updateCheckFiles: Boolean = false
 
-  implicit val summaryReport: SummaryReporting = new SummaryReport(updateCheckFiles = false)
+  implicit val summaryReport: SummaryReporting = new SummaryReport
   implicit def testGroup: TestGroup = TestGroup("VulpixMetaTests")
 
   @Test def compilePos: Unit = compileFilesInDir("tests/vulpix-tests/meta/pos", defaultOptions).checkCompile()

compiler/test/dotty/tools/vulpix/VulpixUnitTests.scala

@@ -21,6 +21,7 @@ class VulpixUnitTests extends ParallelTesting {
   def safeMode = sys.env.get("SAFEMODE").isDefined
   def isInteractive = !sys.env.contains("DRONE")
   def testFilter = None
+  def updateCheckFiles: Boolean = false
 
   // To fail with something else than an AssertionError
   def fail(): Unit = throw new Exception("didn't fail properly")

docs/docs/contributing/testing.md

@@ -80,6 +80,12 @@ This will run both the test `./tests/pos/companions.scala` and
 `./tests/neg/companions.scala` since both of these match the given string.
 This also means that you could run `testCompilation` with no arguments to run all integration tests.
 
+When complex checkfiles must be updated, `testCompilation` can run in a mode where it overrides the checkfiles with the test outputs.
+```bash
+$ sbt
+> testCompilation --update-checkfiles
+```
+
 ### Bootstrapped-only tests
 
 To run `testCompilation` on a bootstrapped Dotty compiler, use

project/Build.scala

@@ -440,10 +440,11 @@ object Build {
 
   def testOnlyFiltered(test: String, options: String) = Def.inputTaskDyn {
     val args = spaceDelimited("<arg>").parsed
-    val cmd = s" $test -- $options" + {
-      if (args.nonEmpty) " -Ddotty.tests.filter=" + args.mkString(" ")
-      else ""
-    }
+    val updateCheckfile = args.contains("--update-checkfiles")
+    val args1 = if (updateCheckfile) args.filter(_ != "--update-checkfiles") else args
+    val cmd = s" $test -- $options" +
+      (if (updateCheckfile) " -Ddotty.tests.updateCheckfiles=true" else "") +
+      (if (args1.nonEmpty) " -Ddotty.tests.filter=" + args1.mkString(" ") else "")
     (testOnly in Test).toTask(cmd)
   }