diff --git a/build_runner_core/lib/src/asset_graph/graph.dart b/build_runner_core/lib/src/asset_graph/graph.dart index 26ef089a7..c516ef5ac 100644 --- a/build_runner_core/lib/src/asset_graph/graph.dart +++ b/build_runner_core/lib/src/asset_graph/graph.dart @@ -70,7 +70,7 @@ class AssetGraph implements GeneratedAssetHider { /// Deserializes this graph. factory AssetGraph.deserialize(List serializedGraph) => - _AssetGraphDeserializer(serializedGraph).deserialize(); + deserializeAssetGraph(serializedGraph); static Future build( BuildPhases buildPhases, @@ -106,7 +106,7 @@ class AssetGraph implements GeneratedAssetHider { return graph; } - List serialize() => _AssetGraphSerializer(this).serialize(); + List serialize() => serializeAssetGraph(this); @visibleForTesting Map>> diff --git a/build_runner_core/lib/src/asset_graph/identity_serializer.dart b/build_runner_core/lib/src/asset_graph/identity_serializer.dart new file mode 100644 index 000000000..dd4bf762c --- /dev/null +++ b/build_runner_core/lib/src/asset_graph/identity_serializer.dart @@ -0,0 +1,91 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'package:built_value/serializer.dart'; + +/// Wraps another `built_value` serializer to add serializing by identity. +/// +/// Integer IDs are created as new values are encountered, and a mapping to +/// objects and serialized object values is maintained. +/// +/// The serialized data does not contain the object values, only IDs. This might +/// not matter if this `IdentitySerializer` will stay in memory and be used to +/// deserialize. Or, the serialized object values must be serialized separately: +/// they can be obtained from [serializedObjects], and set via +/// [deserializeWithObjects]. +/// +/// TODO(davidmorgan): consider upstreaming to `built_value`. +class IdentitySerializer implements PrimitiveSerializer { + final Serializer delegate; + final PrimitiveSerializer? _primitiveDelegate; + final StructuredSerializer? _structuredDelegate; + + final Map _ids = Map.identity(); + final List _objects = []; + List _serializedObjects = []; + + /// A serializer wrapping [delegate] to deduplicate by identity. + IdentitySerializer(this.delegate) + : _primitiveDelegate = delegate is PrimitiveSerializer ? delegate : null, + _structuredDelegate = + delegate is StructuredSerializer ? delegate : null; + + /// Sets the stored object values to [objects]. + /// + /// Serialized values are indices into this list. + void deserializeWithObjects(Iterable objects) { + _ids.clear(); + _objects.clear(); + _serializedObjects.clear(); + for (final object in objects) { + _objects.add(object); + _ids[object] = _objects.length - 1; + } + } + + /// The list of unique objects encountered since the most recent [reset]. + List get serializedObjects => _serializedObjects; + + /// Clears the ID to object and serialized object mappings. + /// + /// Call this after serializing or deserializing to avoid retaining objects in + /// memory; or, don't call it to continue using the same IDs and objects. + void reset() { + _ids.clear(); + _objects.clear(); + _serializedObjects = []; + } + + @override + Iterable get types => delegate.types; + + @override + String get wireName => delegate.wireName; + + @override + T deserialize( + Serializers serializers, + Object serialized, { + FullType specifiedType = FullType.unspecified, + }) => _objects[serialized as int]; + + @override + Object serialize( + Serializers serializers, + T object, { + FullType specifiedType = FullType.unspecified, + }) { + // If it has already been seen, return the ID. + return _ids.putIfAbsent(object, () { + // Otherwise, serialize it, store the value and serialized value, and + // return the index of the last of `_objects` as the ID. + final serialized = + _primitiveDelegate == null + ? _structuredDelegate!.serialize(serializers, object) + : _primitiveDelegate.serialize(serializers, object); + _serializedObjects.add(serialized); + return (_objects..add(object)).length - 1; + }); + } +} diff --git a/build_runner_core/lib/src/asset_graph/serialization.dart b/build_runner_core/lib/src/asset_graph/serialization.dart index be6d45335..4cad1866f 100644 --- a/build_runner_core/lib/src/asset_graph/serialization.dart +++ b/build_runner_core/lib/src/asset_graph/serialization.dart @@ -8,148 +8,95 @@ part of 'graph.dart'; /// /// This should be incremented any time the serialize/deserialize formats /// change. -const _version = 27; +const _version = 28; /// Deserializes an [AssetGraph] from a [Map]. -class _AssetGraphDeserializer { - // Iteration order does not matter - final _idToAssetId = HashMap(); - final Map _serializedGraph; - - _AssetGraphDeserializer._(this._serializedGraph); - - factory _AssetGraphDeserializer(List bytes) { - dynamic decoded; - try { - decoded = jsonDecode(utf8.decode(bytes)); - } on FormatException { - throw AssetGraphCorruptedException(); - } - if (decoded is! Map) throw AssetGraphCorruptedException(); - if (decoded['version'] != _version) { - throw AssetGraphCorruptedException(); - } - return _AssetGraphDeserializer._(decoded); +AssetGraph deserializeAssetGraph(List bytes) { + dynamic serializedGraph; + try { + serializedGraph = jsonDecode(utf8.decode(bytes)); + } on FormatException { + throw AssetGraphCorruptedException(); + } + if (serializedGraph is! Map) throw AssetGraphCorruptedException(); + if (serializedGraph['version'] != _version) { + throw AssetGraphCorruptedException(); } - /// Perform the deserialization, should only be called once. - AssetGraph deserialize() { - var packageLanguageVersions = { - for (var entry - in (_serializedGraph['packageLanguageVersions'] - as Map) - .entries) - entry.key: - entry.value != null - ? LanguageVersion.parse(entry.value as String) - : null, - }; - var graph = AssetGraph._( - _deserializeDigest(_serializedGraph['buildActionsDigest'] as String)!, - _serializedGraph['dart_version'] as String, - packageLanguageVersions.build(), - BuiltList.from(_serializedGraph['enabledExperiments'] as List), - ); - - var packageNames = _serializedGraph['packages'] as List; - - // Read in the id => AssetId map from the graph first. - var assetPaths = _serializedGraph['assetPaths'] as List; - for (var i = 0; i < assetPaths.length; i += 2) { - var packageName = packageNames[assetPaths[i + 1] as int] as String; - _idToAssetId[i ~/ 2] = AssetId(packageName, assetPaths[i] as String); - } - - // Read in all the nodes and their outputs. - // - // Note that this does not read in the inputs of generated nodes. - for (var serializedItem in _serializedGraph['nodes'] as Iterable) { - graph._add(_deserializeAssetNode(serializedItem as List)); - } + identityAssetIdSerializer.deserializeWithObjects( + (serializedGraph['ids'] as List).map( + (id) => assetIdSerializer.deserialize(serializers, id as Object), + ), + ); + + var packageLanguageVersions = { + for (var entry + in (serializedGraph['packageLanguageVersions'] as Map) + .entries) + entry.key: + entry.value != null + ? LanguageVersion.parse(entry.value as String) + : null, + }; + var graph = AssetGraph._( + _deserializeDigest(serializedGraph['buildActionsDigest'] as String)!, + serializedGraph['dart_version'] as String, + packageLanguageVersions.build(), + BuiltList.from(serializedGraph['enabledExperiments'] as List), + ); + + for (var serializedItem in serializedGraph['nodes'] as Iterable) { + graph._add(_deserializeAssetNode(serializedItem as List)); + } - final postProcessOutputs = - serializers.deserialize( - _serializedGraph['postProcessOutputs'], - specifiedType: postProcessBuildStepOutputsFullType, - ) - as Map>>; + final postProcessOutputs = + serializers.deserialize( + serializedGraph['postProcessOutputs'], + specifiedType: postProcessBuildStepOutputsFullType, + ) + as Map>>; - for (final postProcessOutputsForPackage in postProcessOutputs.values) { - for (final entry in postProcessOutputsForPackage.entries) { - graph.updatePostProcessBuildStep(entry.key, outputs: entry.value); - } + for (final postProcessOutputsForPackage in postProcessOutputs.values) { + for (final entry in postProcessOutputsForPackage.entries) { + graph.updatePostProcessBuildStep(entry.key, outputs: entry.value); } - - return graph; } - AssetNode _deserializeAssetNode(List serializedNode) => - serializers.deserializeWith(AssetNode.serializer, serializedNode) - as AssetNode; + identityAssetIdSerializer.reset(); + return graph; } -/// Serializes an [AssetGraph] into a [Map]. -class _AssetGraphSerializer { - // Iteration order does not matter - final _assetIdToId = HashMap(); - - final AssetGraph _graph; - - _AssetGraphSerializer(this._graph); - - /// Perform the serialization, should only be called once. - List serialize() { - var pathId = 0; - // [path0, packageId0, path1, packageId1, ...] - var assetPaths = []; - var packages = _graph._nodesByPackage.keys.toList(growable: false); - for (var node in _graph.allNodes) { - _assetIdToId[node.id] = pathId; - pathId++; - assetPaths - ..add(node.id.path) - ..add(packages.indexOf(node.id.package)); - } +AssetNode _deserializeAssetNode(List serializedNode) => + serializers.deserializeWith(AssetNode.serializer, serializedNode) + as AssetNode; - var result = { - 'version': _version, - 'dart_version': _graph.dartVersion, - 'nodes': _graph.allNodes - .map((node) => serializers.serializeWith(AssetNode.serializer, node)) - .toList(growable: false), - 'buildActionsDigest': _serializeDigest(_graph.buildPhasesDigest), - 'packages': packages, - 'assetPaths': assetPaths, - 'packageLanguageVersions': - _graph.packageLanguageVersions - .map((pkg, version) => MapEntry(pkg, version?.toString())) - .toMap(), - 'enabledExperiments': _graph.enabledExperiments.toList(), - 'postProcessOutputs': serializers.serialize( - _graph._postProcessBuildStepOutputs, - specifiedType: postProcessBuildStepOutputsFullType, - ), - }; - return utf8.encode(json.encode(result)); - } - - int findAssetIndex( - AssetId id, { - required AssetId from, - required String field, - }) { - final index = _assetIdToId[id]; - if (index == null) { - log.severe( - 'The $field field in $from references a non-existent asset ' - '$id and will corrupt the asset graph. ' - 'If you encounter this error please copy ' - 'the details from this message and add them to ' - 'https://github.com/dart-lang/build/issues/1804.', - ); - } - return index!; - } +/// Serializes an [AssetGraph] into a [Map]. +List serializeAssetGraph(AssetGraph graph) { + // Serialize nodes first so all `AssetId` instances are seen by + // `identityAssetIdSeralizer`. + final nodes = graph.allNodes + .map((node) => serializers.serializeWith(AssetNode.serializer, node)) + .toList(growable: false); + + var result = { + 'version': _version, + 'ids': identityAssetIdSerializer.serializedObjects, + 'dart_version': graph.dartVersion, + 'nodes': nodes, + 'buildActionsDigest': _serializeDigest(graph.buildPhasesDigest), + 'packageLanguageVersions': + graph.packageLanguageVersions + .map((pkg, version) => MapEntry(pkg, version?.toString())) + .toMap(), + 'enabledExperiments': graph.enabledExperiments.toList(), + 'postProcessOutputs': serializers.serialize( + graph._postProcessBuildStepOutputs, + specifiedType: postProcessBuildStepOutputsFullType, + ), + }; + + identityAssetIdSerializer.reset(); + return utf8.encode(json.encode(result)); } Digest? _deserializeDigest(String? serializedDigest) => diff --git a/build_runner_core/lib/src/asset_graph/serializers.dart b/build_runner_core/lib/src/asset_graph/serializers.dart index 40e8b316f..a9eabdcce 100644 --- a/build_runner_core/lib/src/asset_graph/serializers.dart +++ b/build_runner_core/lib/src/asset_graph/serializers.dart @@ -9,6 +9,7 @@ import 'package:built_collection/built_collection.dart'; import 'package:built_value/serializer.dart'; import 'package:crypto/crypto.dart'; +import 'identity_serializer.dart'; import 'node.dart'; import 'post_process_build_step_id.dart'; @@ -24,10 +25,15 @@ final postProcessBuildStepOutputsFullType = FullType(Map, [ postProcessBuildStepOutputsInnerFullType, ]); +final assetIdSerializer = AssetIdSerializer(); +final identityAssetIdSerializer = IdentitySerializer( + assetIdSerializer, +); + @SerializersFor([AssetNode]) final Serializers serializers = (_$serializers.toBuilder() - ..add(AssetIdSerializer()) + ..add(identityAssetIdSerializer) ..add(DigestSerializer()) ..add(MapSerializer()) ..add(SetSerializer())