have build/test scripts automatically update baselines

we already have to review them, so running a diff and failing the rest of the test pass seems unhelpful. Instead operate like our generated SDK does, and update in place

[email protected]

Review URL: https://codereview.chromium.org/1188173003.

Author: John Messerly

pkg/dev_compiler/.gitignore

@@ -3,7 +3,7 @@
 build/
 packages
 
-# Or the files created by tools.
+# Ignore files created by tools.
 *.dart.precompiled.js
 *.js_
 *.js.deps
@@ -12,11 +12,12 @@ packages
 .pub/
 node_modules
 
+# Ignore test output
+test/codegen/expect/dev_compiler/
+test/codegen/expect/sunflower/dev_compiler/
+
 # Created by ./tool/build_sdk.sh
 tool/generated_sdk/
 
-# Or our test outputs
-test/codegen/actual/
-
 # Include when developing application packages.
 pubspec.lock

pkg/dev_compiler/test/codegen_test.dart

@@ -49,7 +49,7 @@ main(arguments) {
   });
 
   var inputDir = path.join(testDirectory, 'codegen');
-  var actualDir = path.join(inputDir, 'actual');
+  var expectDir = path.join(inputDir, 'expect');
   var paths = new Directory(inputDir)
       .listSync()
       .where((f) => f is File)
@@ -66,8 +66,8 @@ main(arguments) {
             entryPointFile: entryPoint, dartSdkPath: sdkPath),
         codegenOptions: new CodegenOptions(
             outputDir: subDir == null
-                ? actualDir
-                : path.join(actualDir, subDir),
+                ? expectDir
+                : path.join(expectDir, subDir),
             emitSourceMaps: sourceMaps,
             forceCompile: checkSdk),
         useColors: false,
@@ -80,7 +80,7 @@ main(arguments) {
   var realSdk = getSdkDir(arguments).path;
 
   // Remove old output, and `packages` symlinks which mess up the diff.
-  var dir = new Directory(actualDir);
+  var dir = new Directory(expectDir);
   if (dir.existsSync()) dir.deleteSync(recursive: true);
   var packagesDirs = new Directory(inputDir)
       .listSync(recursive: true)
@@ -100,10 +100,10 @@ main(arguments) {
       var success = !result.failure;
 
       // Write compiler messages to disk.
-      new File(path.join(actualDir, '$filename.txt'))
+      new File(path.join(expectDir, '$filename.txt'))
           .writeAsStringSync(compilerMessages.toString());
 
-      var outFile = new File(path.join(actualDir, '$filename.js'));
+      var outFile = new File(path.join(expectDir, '$filename.js'));
       expect(outFile.existsSync(), success,
           reason: '${outFile.path} was created iff compilation succeeds');
 
@@ -133,8 +133,8 @@ main(arguments) {
         // be generated against a specific SDK version.
         var testSdk = path.join(testDirectory, 'generated_sdk');
         var result = compile('dart:core', testSdk, checkSdk: true);
-        var outputDir = new Directory(path.join(actualDir, 'core'));
-        var outFile = new File(path.join(actualDir, 'dart/core.js'));
+        var outputDir = new Directory(path.join(expectDir, 'core'));
+        var outFile = new File(path.join(expectDir, 'dart/core.js'));
         expect(outFile.existsSync(), true,
             reason: '${outFile.path} was created for dart:core');
       });
@@ -152,7 +152,7 @@ main(arguments) {
     var success = !result.failure;
 
     // Write compiler messages to disk.
-    new File(path.join(actualDir, 'sunflower', 'sunflower.txt'))
+    new File(path.join(expectDir, 'sunflower', 'sunflower.txt'))
         .writeAsStringSync(compilerMessages.toString());
 
     var expectedFiles = [
@@ -163,7 +163,7 @@ main(arguments) {
     ]..addAll(expectedRuntime);
 
     for (var filepath in expectedFiles) {
-      var outFile = new File(path.join(actualDir, 'sunflower', filepath));
+      var outFile = new File(path.join(expectDir, 'sunflower', filepath));
       expect(outFile.existsSync(), success,
           reason: '${outFile.path} was created iff compilation succeeds');
     }
@@ -177,7 +177,7 @@ main(arguments) {
     var success = !result.failure;
 
     // Write compiler messages to disk.
-    new File(path.join(actualDir, 'html_input.txt'))
+    new File(path.join(expectDir, 'html_input.txt'))
         .writeAsStringSync(compilerMessages.toString());
 
     var expectedFiles = [
@@ -190,7 +190,7 @@ main(arguments) {
     ]..addAll(expectedRuntime);
 
     for (var filepath in expectedFiles) {
-      var outFile = new File(path.join(actualDir, filepath));
+      var outFile = new File(path.join(expectDir, filepath));
       expect(outFile.existsSync(), success,
           reason: '${outFile.path} was created iff compilation succeeds');
     }
@@ -200,7 +200,7 @@ main(arguments) {
       'dev_compiler/runtime/messages.css'
     ];
     for (var filepath in notExpectedFiles) {
-      var outFile = new File(path.join(actualDir, filepath));
+      var outFile = new File(path.join(expectDir, filepath));
       expect(outFile.existsSync(), isFalse,
           reason: '${outFile.path} should only be generated in server mode');
     }
@@ -215,7 +215,7 @@ main(arguments) {
     var success = !result.failure;
 
     // Write compiler messages to disk.
-    new File(path.join(actualDir, 'server_mode', 'html_input.txt'))
+    new File(path.join(expectDir, 'server_mode', 'html_input.txt'))
         .writeAsStringSync(compilerMessages.toString());
 
     var expectedFiles = [
@@ -229,11 +229,11 @@ main(arguments) {
     ]..addAll(expectedRuntime);
 
     // Parse the HTML file and verify its contents were expected.
-    var htmlPath = path.join(actualDir, 'server_mode', 'html_input.html');
+    var htmlPath = path.join(expectDir, 'server_mode', 'html_input.html');
     var doc = html.parse(new File(htmlPath).readAsStringSync());
 
     for (var filepath in expectedFiles) {
-      var outPath = path.join(actualDir, 'server_mode', filepath);
+      var outPath = path.join(expectDir, 'server_mode', filepath);
       expect(new File(outPath).existsSync(), success,
           reason: '$outPath was created iff compilation succeeds');
 
@@ -257,7 +257,7 @@ main(arguments) {
     }
 
     // Clean up the server mode folder, otherwise it causes diff churn.
-    var dir = new Directory(path.join(actualDir, 'server_mode'));
+    var dir = new Directory(path.join(expectDir, 'server_mode'));
     if (dir.existsSync()) dir.deleteSync(recursive: true);
   });
 }

pkg/dev_compiler/tool/build_sdk.sh

@@ -17,22 +17,10 @@ fi
 # https://github.com/dart-lang/dev_compiler/issues/219
 dart -c bin/devc.dart --no-source-maps --sdk-check --force-compile -l warning \
     --dart-sdk tool/generated_sdk -o lib/runtime/ dart:mirrors \
-    > tool/generated_sdk/sdk_errors.txt || true
+    > tool/sdk_expected_errors.txt || true
 
 if [[ ! -f lib/runtime/dart/core.js ]] ; then
     echo 'core.js not found, assuming build failed.'
     echo './tool/build_sdk.sh can be run to reproduce this.'
     exit 1
 fi
-
-DIFF_ARGS="-u tool/sdk_expected_errors.txt tool/generated_sdk/sdk_errors.txt"
-
-if ! (diff $DIFF_ARGS > /dev/null) ; then
-    diff $DIFF_ARGS |\
-        sed -e "s/^\(+.*\)/\1/" |\
-        sed -e "s/^\(-.*\)/\1/"
-    echo
-    echo 'SDK errors have changed.  To update expectations, run:'
-    echo '$ cp tool/generated_sdk/sdk_errors.txt tool/sdk_expected_errors.txt'
-    exit 1
-fi

pkg/dev_compiler/tool/test.sh

@@ -6,24 +6,6 @@ function fail {
   return 1
 }
 
-# Arguments passed to the diff tool. We exclude:
-#  - *.map files so they aren't compared, as the diff is not human readable.
-#  - runtime JS files that are just copied over from the sources and are not
-#    duplicated in the expected folder.
-DIFF_ARGS="-u -r -N --exclude=\*.map expect actual"
-
-function show_diff {
-  echo "Fail: actual output did not match expected"
-  echo
-  diff $DIFF_ARGS |\
-    sed -e "s/^\(+.*\)/\1/" |\
-    sed -e "s/^\(-.*\)/\1/"
-  echo
-  echo "You can update these expectations with:"
-  echo "$ pushd `pwd` && cp -a actual/* expect && popd"
-  fail
-}
-
 # Some tests require being run from the package root
 # switch to the root directory of dev_compiler
 cd $( dirname "${BASH_SOURCE[0]}" )/..
@@ -40,12 +22,6 @@ cd $( dirname "${BASH_SOURCE[0]}" )/..
 unset COVERALLS_TOKEN
 pub run test:test test/all_tests.dart || fail
 
-# validate codegen_test output
-pushd test/codegen/ &> /dev/null
-rm -r actual/dev_compiler/ actual/sunflower/dev_compiler
-diff $DIFF_ARGS > /dev/null || show_diff
-popd &> /dev/null
-
 # run self host and analyzer after other tests, because they're ~seconds to run.
 pub run test:test test/checker/self_host_test.dart || fail