[ddc] Remove unsound .dill files from SDK

Dart 3 requires all code to be null safe.

Compiling with --no-sound-null safety requires manually passing
an unsound SDK outline. This is no longer provided by default
and is not packaged with the SDK at all.

Fixes: #50700
Change-Id: Ic4bd7dc5e96b42c7c10a29f273371c6a680936f1
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/278999
Reviewed-by: William Hesse <[email protected]>
Commit-Queue: Nicholas Shahan <[email protected]>

Author: nshahan

pkg/dev_compiler/lib/src/kernel/command.dart

@@ -140,6 +140,27 @@ Future<CompilerResult> _compile(List<String> args,
 
   var options = SharedCompilerOptions.fromArguments(argResults);
 
+  Uri toCustomUri(Uri uri) {
+    if (!uri.hasScheme) {
+      return Uri(scheme: options.multiRootScheme, path: '/${uri.path}');
+    }
+    return uri;
+  }
+
+  // TODO(jmesserly): this is a workaround for the CFE, which does not
+  // understand relative URIs, and we'd like to avoid absolute file URIs
+  // being placed in the summary if possible.
+  // TODO(jmesserly): investigate if Analyzer has a similar issue.
+  Uri sourcePathToCustomUri(String source) {
+    return toCustomUri(sourcePathToRelativeUri(source));
+  }
+
+  // Compile SDK module directly from a provided .dill file.
+  var inputs = [for (var arg in argResults.rest) sourcePathToCustomUri(arg)];
+  if (inputs.length == 1 && inputs.single.path.endsWith('.dill')) {
+    return compileSdkFromDill(args);
+  }
+
   // To make the output .dill agnostic of the current working directory,
   // we use a custom-uri scheme for all app URIs (these are files outside the
   // lib folder). The following [FileSystem] will resolve those references to
@@ -162,44 +183,36 @@ Future<CompilerResult> _compile(List<String> args,
 
   var fileSystem = MultiRootFileSystem(
       options.multiRootScheme, multiRootPaths, fe.StandardFileSystem.instance);
-
-  Uri toCustomUri(Uri uri) {
-    if (!uri.hasScheme) {
-      return Uri(scheme: options.multiRootScheme, path: '/${uri.path}');
-    }
-    return uri;
-  }
-
-  // TODO(jmesserly): this is a workaround for the CFE, which does not
-  // understand relative URIs, and we'd like to avoid absolute file URIs
-  // being placed in the summary if possible.
-  // TODO(jmesserly): investigate if Analyzer has a similar issue.
-  Uri sourcePathToCustomUri(String source) {
-    return toCustomUri(sourcePathToRelativeUri(source));
-  }
-
   var summaryPaths = options.summaryModules.keys.toList();
   var summaryModules = Map.fromIterables(
       summaryPaths.map(sourcePathToUri).cast<Uri>(),
       options.summaryModules.values);
   var sdkSummaryPath = argResults['dart-sdk-summary'] as String?;
   var librarySpecPath = argResults['libraries-file'] as String?;
+  var compileSdk = argResults['compile-sdk'] == true;
   if (sdkSummaryPath == null) {
-    sdkSummaryPath =
-        defaultSdkSummaryPath(soundNullSafety: options.soundNullSafety);
-    librarySpecPath ??= defaultLibrarySpecPath;
+    if (!compileSdk) {
+      if (!options.soundNullSafety) {
+        // Technically if you can produce an SDK outline .dill and pass it
+        // this error can be avoided and the compile will still work for now.
+        // If you are reading this comment be warned, this loophole will be
+        // removed without warning in the future.
+        print('Dart 3 only supports sound null safety, '
+            'see https://dart.dev/null-safety.\n');
+        return CompilerResult(64);
+      }
+      sdkSummaryPath = defaultSdkSummaryPath;
+      librarySpecPath ??= defaultLibrarySpecPath;
+    }
+    // Compiling without manually passing or getting a default SDK summary is
+    // only allowed when `compileSdk` is true.
   }
   var invalidSummary = summaryPaths.any((s) => !s.endsWith('.dill')) ||
-      !sdkSummaryPath.endsWith('.dill');
+      (sdkSummaryPath != null && !sdkSummaryPath.endsWith('.dill'));
   if (invalidSummary) {
     throw StateError('Non-dill file detected in input: $summaryPaths');
   }
 
-  var inputs = [for (var arg in argResults.rest) sourcePathToCustomUri(arg)];
-  if (inputs.length == 1 && inputs.single.path.endsWith('.dill')) {
-    return compileSdkFromDill(args);
-  }
-
   if (librarySpecPath == null) {
     // TODO(jmesserly): the `isSupported` bit should be included in the SDK
     // summary, but front_end requires a separate file, so we have to work
@@ -210,10 +223,12 @@ Future<CompilerResult> _compile(List<String> args,
     // (if the user is doing something custom).
     //
     // Another option: we could make an in-memory file with the relevant info.
-    librarySpecPath =
-        p.join(p.dirname(p.dirname(sdkSummaryPath)), 'libraries.json');
+    librarySpecPath = p.join(
+        p.dirname(p.dirname(sdkSummaryPath ?? defaultSdkSummaryPath)),
+        'libraries.json');
     if (!File(librarySpecPath).existsSync()) {
-      librarySpecPath = p.join(p.dirname(sdkSummaryPath), 'libraries.json');
+      librarySpecPath = p.join(
+          p.dirname(sdkSummaryPath ?? defaultSdkSummaryPath), 'libraries.json');
     }
   }
 
@@ -245,8 +260,6 @@ Future<CompilerResult> _compile(List<String> args,
       onError: stderr.writeln, onWarning: print);
 
   var trackWidgetCreation = argResults['track-widget-creation'] as bool;
-
-  var compileSdk = argResults['compile-sdk'] == true;
   var oldCompilerState = compilerState;
   var recordUsedInputs = argResults['used-inputs-file'] != null;
   var additionalDills = summaryModules.keys.toList();
@@ -262,7 +275,7 @@ Future<CompilerResult> _compile(List<String> args,
         oldCompilerState,
         compileSdk,
         sourcePathToUri(getSdkPath()),
-        compileSdk ? null : sourcePathToUri(sdkSummaryPath),
+        compileSdk ? null : sourcePathToUri(sdkSummaryPath!),
         packageFile != null ? sourcePathToUri(packageFile) : null,
         sourcePathToUri(librarySpecPath),
         additionalDills,
@@ -284,7 +297,7 @@ Future<CompilerResult> _compile(List<String> args,
       oldCompilerState = null;
 
       if (!compileSdk) {
-        inputDigests[sourcePathToUri(sdkSummaryPath)] = const [0];
+        inputDigests[sourcePathToUri(sdkSummaryPath!)] = const [0];
       }
       for (var uri in summaryModules.keys) {
         inputDigests[uri] = const [0];
@@ -302,7 +315,7 @@ Future<CompilerResult> _compile(List<String> args,
         doneAdditionalDills,
         compileSdk,
         sourcePathToUri(getSdkPath()),
-        compileSdk ? null : sourcePathToUri(sdkSummaryPath),
+        compileSdk ? null : sourcePathToUri(sdkSummaryPath!),
         packageFile != null ? sourcePathToUri(packageFile) : null,
         sourcePathToUri(librarySpecPath),
         additionalDills,
@@ -317,8 +330,9 @@ Future<CompilerResult> _compile(List<String> args,
         nnbdMode:
             options.soundNullSafety ? fe.NnbdMode.Strong : fe.NnbdMode.Weak);
     var incrementalCompiler = compilerState.incrementalCompiler!;
-    var cachedSdkInput =
-        compilerState.workerInputCache![sourcePathToUri(sdkSummaryPath)];
+    var cachedSdkInput = compileSdk
+        ? null
+        : compilerState.workerInputCache![sourcePathToUri(sdkSummaryPath!)];
     compilerState.options.onDiagnostic = diagnosticMessageHandler;
     var incrementalCompilerResult = await incrementalCompiler.computeDelta(
         entryPoints: inputs,
@@ -819,13 +833,9 @@ Map<String, String> parseAndRemoveDeclaredVariables(List<String> args) {
   return declaredVariables;
 }
 
-/// The default path of the kernel summary for the Dart SDK given the
-/// [soundNullSafety] mode.
-String defaultSdkSummaryPath({required bool soundNullSafety}) {
-  var outlineDill =
-      soundNullSafety ? 'ddc_outline.dill' : 'ddc_outline_unsound.dill';
-  return p.join(getSdkPath(), 'lib', '_internal', outlineDill);
-}
+/// The default path of the kernel summary for the Dart SDK.
+final defaultSdkSummaryPath =
+    p.join(getSdkPath(), 'lib', '_internal', 'ddc_outline.dill');
 
 final defaultLibrarySpecPath = p.join(getSdkPath(), 'lib', 'libraries.json');
 

pkg/dev_compiler/test/expression_compiler/expression_compiler_e2e_suite.dart

@@ -54,10 +54,15 @@ class DevelopmentIncrementalCompiler extends fe.IncrementalCompiler {
 
 class SetupCompilerOptions {
   static final sdkRoot = fe.computePlatformBinariesLocation();
+  static final buildRoot =
+      fe.computePlatformBinariesLocation(forceBuildDir: true);
+  // Unsound .dill files are not longer in the released SDK so this file must be
+  // read from the build output directory.
   static final sdkUnsoundSummaryPath =
-      p.join(sdkRoot.toFilePath(), 'ddc_outline_unsound.dill');
+      buildRoot.resolve('ddc_outline_unsound.dill').toFilePath();
+  // Use the outline copied to the released SDK.
   static final sdkSoundSummaryPath =
-      p.join(sdkRoot.toFilePath(), 'ddc_outline.dill');
+      sdkRoot.resolve('ddc_outline.dill').toFilePath();
   static final librariesSpecificationUri =
       p.join(p.dirname(p.dirname(getSdkPath())), 'libraries.json');
 
@@ -301,7 +306,6 @@ class TestDriver {
     this.setup = setup;
     this.source = source;
     testDir = chromeDir.createTempSync('ddc_eval_test');
-    var buildDir = p.dirname(p.dirname(p.dirname(Platform.resolvedExecutable)));
     var scriptPath = Platform.script.normalizePath().toFilePath();
     var ddcPath = p.dirname(p.dirname(p.dirname(scriptPath)));
     output = testDir.uri.resolve('test.js');
@@ -340,14 +344,15 @@ class TestDriver {
 
     switch (setup.moduleFormat) {
       case ModuleFormat.ddc:
-        var dartSdkPath = escaped(p.join(
-            buildDir,
-            'gen',
-            'utils',
-            'dartdevc',
-            setup.soundNullSafety ? 'sound' : 'kernel',
-            'legacy',
-            'dart_sdk.js'));
+        var dartSdkPath = escaped(SetupCompilerOptions.buildRoot
+            .resolve(p.join(
+                'gen',
+                'utils',
+                'dartdevc',
+                setup.soundNullSafety ? 'sound' : 'kernel',
+                'legacy',
+                'dart_sdk.js'))
+            .toFilePath());
         if (!File(dartSdkPath).existsSync()) {
           throw Exception('Unable to find Dart SDK at $dartSdkPath');
         }
@@ -375,13 +380,17 @@ class TestDriver {
 ''');
         break;
       case ModuleFormat.amd:
-        var dartSdkPath = escaped(p.join(buildDir, 'gen', 'utils', 'dartdevc',
-            setup.soundNullSafety ? 'sound' : 'kernel', 'amd', 'dart_sdk'));
+        var dartSdkPath = escaped(SetupCompilerOptions.buildRoot
+            .resolve(p.join('gen', 'utils', 'dartdevc',
+                setup.soundNullSafety ? 'sound' : 'kernel', 'amd', 'dart_sdk'))
+            .toFilePath());
         if (!File('$dartSdkPath.js').existsSync()) {
           throw Exception('Unable to find Dart SDK at $dartSdkPath.js');
         }
-        var requirePath = escaped(p.join(
-            buildDir, 'dart-sdk', 'lib', 'dev_compiler', 'amd', 'require.js'));
+        var requirePath = escaped(SetupCompilerOptions.buildRoot
+            .resolve(
+                p.join('dart-sdk', 'lib', 'dev_compiler', 'amd', 'require.js'))
+            .toFilePath());
         var outputPath = escaped(p.withoutExtension(output.toFilePath()));
         bootstrapFile.writeAsStringSync('''
 <script src='$requirePath'></script>

pkg/dev_compiler/test/expression_compiler/expression_compiler_worker_shared.dart

@@ -737,9 +737,13 @@ class TestProjectConfiguration {
   Uri get packagesPath => root.resolve('package_config.json');
 
   Uri get sdkRoot => computePlatformBinariesLocation();
+  // Use the outline copied to the released SDK.
+  // Unsound .dill files are not longer in the released SDK so this file must be
+  // read from the build output directory.
   Uri get sdkSummaryPath => soundNullSafety
       ? sdkRoot.resolve('ddc_outline.dill')
-      : sdkRoot.resolve('ddc_outline_unsound.dill');
+      : computePlatformBinariesLocation(forceBuildDir: true)
+          .resolve('ddc_outline_unsound.dill');
   Uri get librariesPath => sdkRoot.resolve('lib/libraries.json');
 
   List get inputUris => [
@@ -1139,7 +1143,7 @@ class DDCKernelGenerator {
     for (var module in config.modules.values) {
       exitCode = await _generateSummary(module);
       expect(exitCode, 0,
-          reason: 'Failed to generate fsummary for ${module.moduleName}');
+          reason: 'Failed to generate summary dill for ${module.moduleName}');
     }
 
     // generate full dill

pkg/dev_compiler/test/expression_compiler/scope_offset_test.dart

@@ -14,7 +14,10 @@ import 'package:test/test.dart';
 bool get verbose => false;
 
 Uri get sdkRoot => computePlatformBinariesLocation();
-Uri get sdkSummaryPath => sdkRoot.resolve('ddc_outline_unsound.dill');
+// Unsound .dill files are not longer in the released SDK so this file must be
+// read from the build output directory.
+Uri get sdkSummaryPath => computePlatformBinariesLocation(forceBuildDir: true)
+    .resolve('ddc_outline_unsound.dill');
 Uri get librariesPath => sdkRoot.resolve('lib/libraries.json');
 
 void main(List<String> args) {

pkg/dev_compiler/test/nullable_inference_test.dart

@@ -11,6 +11,7 @@ import 'package:dev_compiler/src/kernel/js_typerep.dart';
 import 'package:dev_compiler/src/kernel/nullable_inference.dart';
 import 'package:dev_compiler/src/kernel/target.dart';
 import 'package:front_end/src/api_unstable/ddc.dart' as fe;
+import 'package:front_end/src/compute_platform_binaries_location.dart';
 import 'package:kernel/class_hierarchy.dart';
 import 'package:kernel/core_types.dart';
 import 'package:kernel/kernel.dart';
@@ -657,8 +658,9 @@ Future<CompileResult> kernelCompile(String code) async {
   var sdkUri = Uri.file('/memory/ddc_outline_unsound.dill');
   var sdkFile = _fileSystem.entityForUri(sdkUri);
   if (!await sdkFile.exists()) {
+    var buildRoot = computePlatformBinariesLocation(forceBuildDir: true);
     var outlineDill =
-        p.join(getSdkPath(), 'lib', '_internal', 'ddc_outline_unsound.dill');
+        buildRoot.resolve('ddc_outline_unsound.dill').toFilePath();
     sdkFile.writeAsBytesSync(File(outlineDill).readAsBytesSync());
   }
   var librariesUri = Uri.file('/memory/libraries.json');

pkg/dev_compiler/test/sdk_source_map_test.dart

@@ -9,6 +9,12 @@ import 'package:front_end/src/compute_platform_binaries_location.dart';
 import 'package:path/path.dart' as p;
 import 'package:source_maps/source_maps.dart' as sm;
 
+/// Verifies that the compiled SDK modules used in the SDK test suites have
+/// source maps, and those mappings correctly point to the .dart source files.
+///
+/// There are no longer any precompiled SDK modules distributed with the Dart
+/// SDK so this test depends on built assets from the gen/utils/dartdevc
+/// directory.
 void main() async {
   // This test relies on source maps for the built SDK when working inside the
   // Dart SDK repo.

pkg/dev_compiler/test/shared_test_options.dart

@@ -40,10 +40,13 @@ class DevelopmentIncrementalCompiler extends IncrementalCompiler {
 
 class SetupCompilerOptions {
   static final sdkRoot = computePlatformBinariesLocation();
+  // Unsound .dill files are not longer in the released SDK so this file must be
+  // read from the build output directory.
   static final sdkUnsoundSummaryPath =
-      p.join(sdkRoot.path, 'ddc_outline_unsound.dill');
-  static final sdkSoundSummaryPath = p.join(sdkRoot.path, 'ddc_outline.dill');
-  // TODO(46617) Call getSdkPath() from command.dart instead.
+      computePlatformBinariesLocation(forceBuildDir: true)
+          .resolve('ddc_outline_unsound.dill');
+  // Use the outline copied to the released SDK.
+  static final sdkSoundSummaryPath = sdkRoot.resolve('ddc_outline.dill');
   static final librariesSpecificationUri =
       p.join(p.dirname(p.dirname(getSdkPath())), 'libraries.json');
 
@@ -54,8 +57,8 @@ class SetupCompilerOptions {
       ..target = DevCompilerTarget(TargetFlags())
       ..librariesSpecificationUri = Uri.base.resolve('sdk/lib/libraries.json')
       ..omitPlatform = true
-      ..sdkSummary = sdkRoot.resolve(
-          soundNullSafety ? sdkSoundSummaryPath : sdkUnsoundSummaryPath)
+      ..sdkSummary =
+          soundNullSafety ? sdkSoundSummaryPath : sdkUnsoundSummaryPath
       ..environmentDefines = const {}
       ..nnbdMode = soundNullSafety ? NnbdMode.Strong : NnbdMode.Weak;
     return options;

pkg/test_runner/lib/src/compiler_configuration.dart

@@ -626,9 +626,12 @@ class DevCompilerConfiguration extends CompilerConfiguration {
     // the bootstrapping code, instead of a compiler option.
     var options = sharedOptions.toList();
     options.remove('--null-assertions');
-    if (!_useSdk) {
+    if (!_useSdk || _configuration.nnbdMode != NnbdMode.strong) {
       // If we're testing a built SDK, DDC will find its own summary.
       //
+      // Unsound summary files are not longer bundled with the built SDK so they
+      // must always be specified manually.
+      //
       // For local development we don't have a built SDK yet, so point directly
       // at the built summary file location.
       var sdkSummaryFile = _configuration.nnbdMode == NnbdMode.strong

sdk/BUILD.gn

@@ -36,8 +36,6 @@ declare_args() {
 # ......dart or dart.exe (executable)
 # ......dart.lib (import library for VM native extensions on Windows)
 # ......dartaotruntime or dartaotruntime.exe (executable)
-# ......dart2js
-# ......dartdevc
 # ......utils/gen_snapshot or utils/gen_snapshot.exe (if not on ia32)
 # ......snapshots/
 # ........analysis_server.dart.snapshot
@@ -70,12 +68,20 @@ declare_args() {
 # ........dart2js_server_platform_unsound.dill
 # ........dart2wasm_platform.dill (if not on ia32)
 # ........dart2wasm_outline.dill (if not on ia32)
+# ........ddc_outline.dill
+# ........ddc_platform.dill
 # ........vm_platform_strong.dill
-# ........dev_compiler/
+# ........js_dev_runtime/
+# ........js_runtime/
+# ........js_shared/
 # ......async/
 # ......collection/
 # ......convert/
 # ......core/
+# ......dev_compiler/
+#.........amd/require.js
+#.........web/dart_stack_trace_mapper.js
+# ......developer/
 # ......html/
 # ......_http/
 # ......internal/
@@ -518,13 +524,10 @@ copy("copy_dev_compiler_dills") {
   deps = [
     ":copy_libraries",
     "../utils/dartdevc:dartdevc_platform_sound",
-    "../utils/dartdevc:dartdevc_platform_unsound",
   ]
   sources = [
     "$root_out_dir/ddc_outline.dill",
-    "$root_out_dir/ddc_outline_unsound.dill",
     "$root_out_dir/ddc_platform.dill",
-    "$root_out_dir/ddc_platform_unsound.dill",
   ]
   outputs =
       [ "$root_out_dir/$dart_sdk_output/lib/_internal/{{source_file_part}}" ]