[cfe] Add MacroSerializer

This adds a [MacroSerializer] interface to abstract how URIs are
created for precompiled macro components.

Two strategies are added. The existing solution using a temporary
directory in TempDirMacroSerializer and a new solution using the
recently added (experimental) Isolate.createUriForKernelBlob feature
in IsolateSerializer.

Change-Id: Id7117cc518f09e9b3762d6ca924c788c81fd9ac0
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/234282
Reviewed-by: Jens Johansen <[email protected]>
Commit-Queue: Johnni Winther <[email protected]>

Author: johnniwinther

pkg/front_end/lib/src/api_prototype/compiler_options.dart

@@ -19,6 +19,7 @@ import 'package:kernel/target/targets.dart' show Target;
 import '../base/nnbd_mode.dart';
 
 import '../fasta/kernel/macro.dart';
+import '../macro_serializer.dart';
 import 'experimental_flags.dart'
     show
         AllowedExperimentalFlags,
@@ -137,7 +138,7 @@ class CompilerOptions {
   /// by the macro executor provided by [macroExecutorProvider].
   ///
   /// This is part of the experimental macro feature.
-  Future<Uri> Function(Component)? macroSerializer;
+  MacroSerializer? macroSerializer;
 
   /// Whether to generate code for the SDK.
   ///

pkg/front_end/lib/src/isolate_macro_serializer.dart

@@ -0,0 +1,33 @@
+// Copyright (c) 2022, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:isolate';
+
+import 'package:kernel/kernel.dart';
+import 'macro_serializer.dart';
+
+/// [MacroSerializer] that uses blobs registered with the current [Isolate] to
+/// give access to precompiled macro [Component]s.
+///
+/// This can only be used with the [Isolate]-based macro executor.
+class IsolateMacroSerializer implements MacroSerializer {
+  final List<Uri> _createdUris = [];
+
+  @override
+  Future<void> close() {
+    for (Uri uri in _createdUris) {
+      (Isolate.current as dynamic).unregisterKernelBlobUri(uri);
+    }
+    _createdUris.clear();
+    return new Future.value();
+  }
+
+  @override
+  Future<Uri> createUriForComponent(Component component) {
+    Uri uri = (Isolate.current as dynamic)
+        .createUriForKernelBlob(writeComponentToBytes(component));
+    _createdUris.add(uri);
+    return new Future.value(uri);
+  }
+}

pkg/front_end/lib/src/kernel_generator_impl.dart

@@ -303,8 +303,8 @@ Future<void> _compileMacros(NeededPrecompilations neededPrecompilations,
   precompilationOptions..fileSystem = new HybridFileSystem(fs);
   CompilerResult? compilerResult =
       await kernelForProgramInternal(uri, precompilationOptions);
-  Uri precompiledUri =
-      await options.macroSerializer!(compilerResult!.component!);
+  Uri precompiledUri = await options.macroSerializer!
+      .createUriForComponent(compilerResult!.component!);
   Map<MacroClass, Uri> precompiledMacroUris =
       options.precompiledMacroUris ??= {};
   neededPrecompilations.macroDeclarations

pkg/front_end/lib/src/macro_serializer.dart

@@ -0,0 +1,18 @@
+// Copyright (c) 2022, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'package:kernel/ast.dart';
+
+/// Interface for supporting serialization of [Component]s for macro
+/// precompilation.
+abstract class MacroSerializer {
+  /// Returns a [Uri] that can be accessed by the macro executor.
+  Future<Uri> createUriForComponent(Component component);
+
+  /// Releases all resources of this serializer.
+  ///
+  /// This must be called when the [Uri]s created by [createUriForComponent]
+  /// are no longer needed.
+  Future<void> close();
+}

pkg/front_end/lib/src/temp_dir_macro_serializer.dart

@@ -0,0 +1,43 @@
+// Copyright (c) 2022, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:io';
+
+import 'package:kernel/ast.dart';
+import 'package:kernel/kernel.dart';
+
+import 'fasta/kernel/utils.dart';
+import 'macro_serializer.dart';
+
+/// [MacroSerializer] that uses .dill files stored in a temporary directory to
+/// provided [Uri]s for precompiled macro [Component]s.
+///
+/// This can be used other with the isolate and process based macro executors.
+class TempDirMacroSerializer implements MacroSerializer {
+  final String? name;
+  Directory? tempDirectory;
+  int precompiledCount = 0;
+
+  TempDirMacroSerializer([this.name]);
+
+  Future<Directory> _ensureDirectory() async {
+    return tempDirectory ??= await Directory.systemTemp.createTemp(name);
+  }
+
+  @override
+  Future<Uri> createUriForComponent(Component component) async {
+    Directory directory = await _ensureDirectory();
+    Uri uri =
+        directory.absolute.uri.resolve('macros${precompiledCount++}.dill');
+    await writeComponentToFile(component, uri);
+    return uri;
+  }
+
+  @override
+  Future<void> close() async {
+    try {
+      await tempDirectory?.delete(recursive: true);
+    } catch (_) {}
+  }
+}

pkg/front_end/test/macro_api_test.dart

@@ -2,29 +2,26 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
-import 'dart:io' show Directory, Platform;
+import 'dart:io' show Platform;
 
-import 'package:_fe_analyzer_shared/src/macros/executor/serialization.dart';
 import 'package:_fe_analyzer_shared/src/macros/executor/isolated_executor.dart'
     as isolatedExecutor;
+import 'package:_fe_analyzer_shared/src/macros/executor/serialization.dart';
 import 'package:expect/expect.dart';
 import 'package:front_end/src/api_prototype/experimental_flags.dart';
 import 'package:front_end/src/api_prototype/front_end.dart';
 import 'package:front_end/src/compute_platform_binaries_location.dart';
 import 'package:front_end/src/fasta/kernel/macro.dart';
-import 'package:front_end/src/fasta/kernel/utils.dart';
+import 'package:front_end/src/isolate_macro_serializer.dart';
+import 'package:front_end/src/macro_serializer.dart';
 import 'package:front_end/src/testing/id_testing_helper.dart';
-import 'package:kernel/ast.dart' hide Arguments;
-import 'package:kernel/kernel.dart';
 import 'package:kernel/target/targets.dart';
 import 'package:vm/target/vm.dart';
 
 Future<void> main(List<String> args) async {
   enableMacros = true;
 
-  Directory tempDirectory =
-      await Directory.systemTemp.createTemp('macro_api_test');
-  int precompiledCount = 0;
+  MacroSerializer macroSerializer = new IsolateMacroSerializer();
   try {
     CompilerOptions options = new CompilerOptions();
     options.sdkRoot = computePlatformBinariesLocation();
@@ -37,12 +34,7 @@ Future<void> main(List<String> args) async {
     };
     options.precompiledMacroUris = {};
     options.target = options.macroTarget = new VmTarget(new TargetFlags());
-    options.macroSerializer = (Component component) async {
-      Uri uri = tempDirectory.absolute.uri
-          .resolve('macros${precompiledCount++}.dill');
-      await writeComponentToFile(component, uri);
-      return uri;
-    };
+    options.macroSerializer = macroSerializer;
 
     InternalCompilerResult result = await kernelForProgramInternal(
         Platform.script.resolve(
@@ -51,6 +43,6 @@ Future<void> main(List<String> args) async {
         retainDataForTesting: true) as InternalCompilerResult;
     Expect.isFalse(result.kernelTargetForTesting!.loader.hasSeenError);
   } finally {
-    await tempDirectory.delete(recursive: true);
+    await macroSerializer.close();
   }
 }

pkg/front_end/test/macro_application/macro_application_test.dart

@@ -17,9 +17,10 @@ import 'package:front_end/src/api_prototype/experimental_flags.dart';
 import 'package:front_end/src/fasta/builder/field_builder.dart';
 import 'package:front_end/src/fasta/builder/member_builder.dart';
 import 'package:front_end/src/fasta/kernel/macro.dart';
-import 'package:front_end/src/fasta/kernel/utils.dart';
 import 'package:front_end/src/fasta/source/source_class_builder.dart';
 import 'package:front_end/src/fasta/source/source_library_builder.dart';
+import 'package:front_end/src/macro_serializer.dart';
+import 'package:front_end/src/temp_dir_macro_serializer.dart';
 import 'package:front_end/src/testing/compiler_common.dart';
 import 'package:front_end/src/testing/id_extractor.dart';
 import 'package:front_end/src/testing/id_testing_helper.dart';
@@ -36,8 +37,8 @@ Future<void> main(List<String> args) async {
   bool generateExpectations = args.contains('-g');
   enableMacros = true;
 
-  Directory tempDirectory =
-      await Directory.systemTemp.createTemp('macro_application');
+  MacroSerializer macroSerializer =
+      new TempDirMacroSerializer('macro_application');
   try {
     Directory dataDir =
         new Directory.fromUri(Platform.script.resolve('data/tests'));
@@ -46,23 +47,22 @@ Future<void> main(List<String> args) async {
         createUriForFileName: createUriForFileName,
         onFailure: onFailure,
         runTest: runTestFor(const MacroDataComputer(), [
-          new MacroTestConfig(dataDir, tempDirectory,
+          new MacroTestConfig(dataDir, macroSerializer,
               generateExpectations: generateExpectations)
         ]),
         preserveWhitespaceInAnnotations: true);
   } finally {
-    await tempDirectory.delete(recursive: true);
+    await macroSerializer.close();
   }
 }
 
 class MacroTestConfig extends TestConfig {
   final Directory dataDir;
-  final Directory tempDirectory;
+  final MacroSerializer macroSerializer;
   final bool generateExpectations;
-  int precompiledCount = 0;
   final Map<MacroClass, Uri> precompiledMacroUris = {};
 
-  MacroTestConfig(this.dataDir, this.tempDirectory,
+  MacroTestConfig(this.dataDir, this.macroSerializer,
       {required this.generateExpectations})
       : super(cfeMarker, 'cfe',
             explicitExperimentalFlags: {ExperimentalFlag.macros: true},
@@ -76,12 +76,7 @@ class MacroTestConfig extends TestConfig {
     };
     options.precompiledMacroUris = precompiledMacroUris;
     options.macroTarget = new VmTarget(new TargetFlags());
-    options.macroSerializer = (Component component) async {
-      Uri uri = tempDirectory.absolute.uri
-          .resolve('macros${precompiledCount++}.dill');
-      await writeComponentToFile(component, uri);
-      return uri;
-    };
+    options.macroSerializer = macroSerializer;
   }
 
   @override

pkg/front_end/test/spell_checking_list_code.txt

@@ -123,6 +123,7 @@ bj
 bk
 blindly
 blob
+blobs
 blocking
 bmp
 bn
@@ -1393,6 +1394,7 @@ unparsed
 unpleasant
 unqualified
 unreachable
+unregister
 unseen
 unset
 unshadowed

pkg/kernel/lib/kernel.dart

@@ -13,6 +13,8 @@
 ///
 library kernel;
 
+import 'dart:typed_data';
+
 import 'ast.dart';
 import 'binary/ast_to_binary.dart';
 import 'binary/ast_from_binary.dart';
@@ -61,7 +63,7 @@ Future writeComponentToBinary(Component component, String path) {
   return future;
 }
 
-List<int> writeComponentToBytes(Component component) {
+Uint8List writeComponentToBytes(Component component) {
   BytesSink sink = new BytesSink();
   new BinaryPrinter(sink).writeComponentFile(component);
   return sink.builder.toBytes();

runtime/tests/vm/dart/spawn_uri_from_kernel_blob_test.dart

@@ -8,7 +8,6 @@
 
 import 'dart:io' show Platform;
 import 'dart:isolate' show Isolate, ReceivePort;
-import 'dart:typed_data' show Uint8List;
 
 import "package:expect/expect.dart";
 import 'package:front_end/src/api_unstable/vm.dart'
@@ -33,7 +32,7 @@ main() async {
     };
   final Component component =
       (await kernelForProgram(sourceUri, options))!.component!;
-  final kernelBlob = writeComponentToBytes(component) as Uint8List;
+  final kernelBlob = writeComponentToBytes(component);
 
   final kernelBlobUri =
       (Isolate.current as dynamic).createUriForKernelBlob(kernelBlob);