diff --git a/build/CHANGELOG.md b/build/CHANGELOG.md index 899f9586e..feefa298d 100644 --- a/build/CHANGELOG.md +++ b/build/CHANGELOG.md @@ -8,6 +8,7 @@ - Refactor `MultiPackageAssetReader` to internal `AssetFinder`. - Add internal `Filesystem` that backs `AssetReader` and `AssetWriter` implementations. +- Refactor `CachingAssetReader` to `FilesystemCache`. ## 2.4.2 diff --git a/build/lib/src/builder/build_step_impl.dart b/build/lib/src/builder/build_step_impl.dart index cdaf3e3aa..140ff400c 100644 --- a/build/lib/src/builder/build_step_impl.dart +++ b/build/lib/src/builder/build_step_impl.dart @@ -21,6 +21,7 @@ import '../resource/resource.dart'; import '../state/asset_finder.dart'; import '../state/asset_path_provider.dart'; import '../state/filesystem.dart'; +import '../state/filesystem_cache.dart'; import '../state/input_tracker.dart'; import '../state/reader_state.dart'; import 'build_step.dart'; @@ -83,6 +84,19 @@ class BuildStepImpl implements BuildStep, AssetReaderState { _stageTracker = stageTracker ?? NoOpStageTracker.instance, _reportUnusedAssets = reportUnusedAssets; + @override + BuildStepImpl copyWith({FilesystemCache? cache}) => BuildStepImpl( + inputId, + allowedOutputs, + _reader.copyWith(cache: cache), + _writer, + _resolvers, + _resourceManager, + _resolvePackageConfig, + stageTracker: _stageTracker, + reportUnusedAssets: _reportUnusedAssets, + ); + @override Filesystem get filesystem => _reader.filesystem; diff --git a/build/lib/src/internal.dart b/build/lib/src/internal.dart index f331af190..d9389f096 100644 --- a/build/lib/src/internal.dart +++ b/build/lib/src/internal.dart @@ -9,5 +9,6 @@ library; export 'state/asset_finder.dart'; export 'state/asset_path_provider.dart'; export 'state/filesystem.dart'; +export 'state/filesystem_cache.dart'; export 'state/input_tracker.dart'; export 'state/reader_state.dart'; diff --git a/build/lib/src/state/filesystem.dart b/build/lib/src/state/filesystem.dart index 59ed13138..3ff86a3bd 100644 --- a/build/lib/src/state/filesystem.dart +++ b/build/lib/src/state/filesystem.dart @@ -10,15 +10,39 @@ import 'package:pool/pool.dart'; import '../asset/id.dart'; import 'asset_path_provider.dart'; +import 'filesystem_cache.dart'; /// The filesystem the build is running on. /// /// Methods behave as the `dart:io` methods with the same names, with some -/// exceptions noted. +/// exceptions noted in the docs. +/// +/// Some methods cache, all uses of the cache are noted in the docs. +/// +/// The cache might be a [PassthroughFilesystemCache] in which case it has no +/// effect. +/// +/// TODO(davidmorgan): extend caching to sync methods, deletes, writes. abstract interface class Filesystem { + FilesystemCache get cache; + + /// Returns a new instance with optionally updated [cache]. + Filesystem copyWith({FilesystemCache? cache}); + + /// Whether the file exists. + /// + /// Uses [cache]. Future exists(AssetId id); + /// Reads a file as a string. + /// + /// Uses [cache]. For `utf8`, the `String` is cached; for any other encoding + /// the bytes are cached but the conversion runs on every read. Future readAsString(AssetId id, {Encoding encoding = utf8}); + + /// Reads a file as bytes. + /// + /// Uses [cache]. Future readAsBytes(AssetId id); /// Deletes a file. @@ -62,23 +86,48 @@ abstract interface class Filesystem { /// A filesystem using [assetPathProvider] to map to the `dart:io` filesystem. class IoFilesystem implements Filesystem { + @override + final FilesystemCache cache; + final AssetPathProvider assetPathProvider; /// Pool for async file operations. final _pool = Pool(32); - IoFilesystem({required this.assetPathProvider}); + IoFilesystem({ + required this.assetPathProvider, + this.cache = const PassthroughFilesystemCache(), + }); + + @override + IoFilesystem copyWith({FilesystemCache? cache}) => IoFilesystem( + assetPathProvider: assetPathProvider, + cache: cache ?? this.cache, + ); @override - Future exists(AssetId id) => _pool.withResource(_fileFor(id).exists); + Future exists(AssetId id) => + cache.exists(id, ifAbsent: () => _pool.withResource(_fileFor(id).exists)); @override - Future readAsBytes(AssetId id) => - _pool.withResource(_fileFor(id).readAsBytes); + Future readAsBytes(AssetId id) => cache.readAsBytes( + id, + ifAbsent: () => _pool.withResource(_fileFor(id).readAsBytes), + ); @override - Future readAsString(AssetId id, {Encoding encoding = utf8}) => - _pool.withResource(_fileFor(id).readAsString); + Future readAsString(AssetId id, {Encoding encoding = utf8}) async { + // The cache only directly supports utf8, for other encodings get the + // bytes via the cache then convert. + if (encoding == utf8) { + return cache.readAsString( + id, + ifAbsent: () => _pool.withResource(_fileFor(id).readAsString), + ); + } else { + return encoding.decode(await readAsBytes(id)); + } + } @override void deleteSync(AssetId id) { @@ -146,17 +195,42 @@ class IoFilesystem implements Filesystem { /// An in-memory [Filesystem]. class InMemoryFilesystem implements Filesystem { - final Map> assets = {}; + @override + FilesystemCache cache; + + final Map> assets; + + InMemoryFilesystem({FilesystemCache? cache}) + : cache = cache ?? const PassthroughFilesystemCache(), + assets = {}; + + InMemoryFilesystem._({required this.cache, required this.assets}); @override - Future exists(AssetId id) async => assets.containsKey(id); + InMemoryFilesystem copyWith({FilesystemCache? cache}) => + InMemoryFilesystem._(assets: assets, cache: cache ?? this.cache); @override - Future readAsBytes(AssetId id) async => assets[id] as Uint8List; + Future exists(AssetId id) async => + cache.exists(id, ifAbsent: () async => assets.containsKey(id)); @override - Future readAsString(AssetId id, {Encoding encoding = utf8}) async => - encoding.decode(assets[id] as Uint8List); + Future readAsBytes(AssetId id) async => + cache.readAsBytes(id, ifAbsent: () async => assets[id] as Uint8List); + + @override + Future readAsString(AssetId id, {Encoding encoding = utf8}) async { + // The cache only directly supports utf8, for other encodings get the + // bytes via the cache then convert. + if (encoding == utf8) { + return cache.readAsString( + id, + ifAbsent: () async => encoding.decode(assets[id]!), + ); + } else { + return encoding.decode(await readAsBytes(id)); + } + } @override Future delete(AssetId id) async { diff --git a/build/lib/src/state/filesystem_cache.dart b/build/lib/src/state/filesystem_cache.dart new file mode 100644 index 000000000..f18496172 --- /dev/null +++ b/build/lib/src/state/filesystem_cache.dart @@ -0,0 +1,154 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:async'; +import 'dart:convert'; +import 'dart:typed_data'; + +import '../asset/id.dart'; +import 'lru_cache.dart'; + +/// Cache for file existence and contents. +/// +/// TODO(davidmorgan): benchmark, optimize the caching strategy. +abstract interface class FilesystemCache { + /// Clears all [ids] from all caches. + /// + /// Waits for any pending reads to complete first. + Future invalidate(Iterable ids); + + /// Whether [id] exists. + /// + /// Returns a cached result if available, or caches and returns `ifAbsent()`. + Future exists(AssetId id, {required Future Function() ifAbsent}); + + /// Reads [id] as bytes. + /// + /// Returns a cached result if available, or caches and returns `ifAbsent()`. + Future readAsBytes( + AssetId id, { + required Future Function() ifAbsent, + }); + + /// Reads [id] as a `String`. + /// + /// Returns a cached result if available, or caches and returns `ifAbsent()`. + /// + /// The encoding used is always `utf8`. For other encodings, use + /// [readAsBytes]. + Future readAsString( + AssetId id, { + required Future Function() ifAbsent, + }); +} + +/// [FilesystemCache] that always reads from the underlying source. +class PassthroughFilesystemCache implements FilesystemCache { + const PassthroughFilesystemCache(); + + @override + Future invalidate(Iterable ids) async {} + + @override + Future exists( + AssetId id, { + required Future Function() ifAbsent, + }) => ifAbsent(); + + @override + Future readAsBytes( + AssetId id, { + required Future Function() ifAbsent, + }) => ifAbsent(); + + @override + Future readAsString( + AssetId id, { + required Future Function() ifAbsent, + }) => ifAbsent(); +} + +/// [FilesystemCache] that stores data in memory. +class InMemoryFilesystemCache implements FilesystemCache { + /// Cached results of [readAsBytes]. + final _bytesContentCache = LruCache( + 1024 * 1024, + 1024 * 1024 * 512, + (value) => value.lengthInBytes, + ); + + /// Pending [readAsBytes] operations. + final _pendingBytesContentCache = >{}; + + /// Cached results of [exists]. + /// + /// Don't bother using an LRU cache for this since it's just booleans. + final _canReadCache = >{}; + + /// Cached results of [readAsString]. + /// + /// These are computed and stored lazily using [readAsBytes]. + /// + /// Only files read with [utf8] encoding (the default) will ever be cached. + final _stringContentCache = LruCache( + 1024 * 1024, + 1024 * 1024 * 512, + (value) => value.length, + ); + + /// Pending `readAsString` operations. + final _pendingStringContentCache = >{}; + + @override + Future invalidate(Iterable ids) async { + // First finish all pending operations, as they will write to the cache. + for (var id in ids) { + await _canReadCache.remove(id); + await _pendingBytesContentCache.remove(id); + await _pendingStringContentCache.remove(id); + } + for (var id in ids) { + _bytesContentCache.remove(id); + _stringContentCache.remove(id); + } + } + + @override + Future exists( + AssetId id, { + required Future Function() ifAbsent, + }) => _canReadCache.putIfAbsent(id, ifAbsent); + + @override + Future readAsBytes( + AssetId id, { + required Future Function() ifAbsent, + }) { + var cached = _bytesContentCache[id]; + if (cached != null) return Future.value(cached); + + return _pendingBytesContentCache.putIfAbsent(id, () async { + final result = await ifAbsent(); + _bytesContentCache[id] = result; + unawaited(_pendingBytesContentCache.remove(id)); + return result; + }); + } + + @override + Future readAsString( + AssetId id, { + required Future Function() ifAbsent, + }) async { + var cached = _stringContentCache[id]; + if (cached != null) return cached; + + return _pendingStringContentCache.putIfAbsent(id, () async { + final result = await ifAbsent(); + _stringContentCache[id] = result; + unawaited(_pendingStringContentCache.remove(id)); + return result; + }); + } +} diff --git a/build_runner_core/lib/src/asset/lru_cache.dart b/build/lib/src/state/lru_cache.dart similarity index 100% rename from build_runner_core/lib/src/asset/lru_cache.dart rename to build/lib/src/state/lru_cache.dart diff --git a/build/lib/src/state/reader_state.dart b/build/lib/src/state/reader_state.dart index 16469c690..bf581ea0f 100644 --- a/build/lib/src/state/reader_state.dart +++ b/build/lib/src/state/reader_state.dart @@ -6,10 +6,17 @@ import '../asset/reader.dart'; import 'asset_finder.dart'; import 'asset_path_provider.dart'; import 'filesystem.dart'; +import 'filesystem_cache.dart'; import 'input_tracker.dart'; /// Provides access to the state backing an [AssetReader]. extension AssetReaderStateExtension on AssetReader { + /// Returns a new instance with optionally updated [cache]. + AssetReader copyWith({FilesystemCache? cache}) { + _requireIsAssetReaderState(); + return (this as AssetReaderState).copyWith(cache: cache); + } + Filesystem get filesystem { _requireIsAssetReaderState(); return (this as AssetReaderState).filesystem; @@ -52,6 +59,9 @@ extension AssetReaderStateExtension on AssetReader { /// The state backing an [AssetReader]. abstract interface class AssetReaderState { + /// Returns a new instance with optionally updated [cache]. + AssetReader copyWith({FilesystemCache? cache}); + /// The [Filesystem] that this reader reads from. /// /// Warning: this access to the filesystem bypasses reader functionality diff --git a/build/test/builder/build_step_impl_test.dart b/build/test/builder/build_step_impl_test.dart index bf7bd669e..3ea65cd0e 100644 --- a/build/test/builder/build_step_impl_test.dart +++ b/build/test/builder/build_step_impl_test.dart @@ -32,7 +32,7 @@ void main() { late List outputs; setUp(() { - var reader = StubAssetReader(); + var reader = InMemoryAssetReaderWriter(); var writer = const StubAssetWriter(); primary = makeAssetId(); outputs = List.generate(5, (index) => makeAssetId()); @@ -175,7 +175,7 @@ void main() { buildStep = BuildStepImpl( primary, [outputId], - StubAssetReader(), + InMemoryAssetReaderWriter(), assetWriter, AnalyzerResolvers.custom(), resourceManager, @@ -236,7 +236,7 @@ void main() { late AssetId output; setUp(() { - var reader = StubAssetReader(); + var reader = InMemoryAssetReaderWriter(); var writer = const StubAssetWriter(); primary = makeAssetId(); output = makeAssetId(); @@ -259,7 +259,7 @@ void main() { }); test('reportUnusedAssets forwards calls if provided', () { - var reader = StubAssetReader(); + var reader = InMemoryAssetReaderWriter(); var writer = const StubAssetWriter(); var unused = {}; var buildStep = BuildStepImpl( diff --git a/build/test/state/filesystem_cache_test.dart b/build/test/state/filesystem_cache_test.dart new file mode 100644 index 000000000..1546cf7c8 --- /dev/null +++ b/build/test/state/filesystem_cache_test.dart @@ -0,0 +1,102 @@ +// Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:typed_data'; + +import 'package:build/build.dart'; +import 'package:build/src/state/filesystem_cache.dart'; +import 'package:test/test.dart'; + +void main() { + final txt1 = AssetId('a', 'foo.txt'); + final txt2 = AssetId('a', 'missing.txt'); + + final txt1Bytes = Uint8List.fromList([1, 2, 3]); + final txt2Bytes = Uint8List.fromList([4, 5, 6]); + + late FilesystemCache cache; + + setUp(() { + cache = InMemoryFilesystemCache(); + }); + + group('exists', () { + test('reads from ifAbsent', () async { + expect(await cache.exists(txt1, ifAbsent: () async => true), isTrue); + expect(await cache.exists(txt2, ifAbsent: () async => false), isFalse); + }); + + test('does not re-read from ifAbsent', () async { + expect(await cache.exists(txt1, ifAbsent: () async => true), isTrue); + expect( + await cache.exists(txt1, ifAbsent: () async => false), + isTrue /* cached value */, + ); + }); + + test('can be invalidated with invalidate', () async { + expect(await cache.exists(txt1, ifAbsent: () async => true), isTrue); + await cache.invalidate([txt1]); + expect( + await cache.exists(txt1, ifAbsent: () async => false), + isFalse /* updated value */, + ); + }); + }); + + group('readAsBytes', () { + test('reads from ifAbsent', () async { + expect( + await cache.readAsBytes(txt1, ifAbsent: () async => txt1Bytes), + txt1Bytes, + ); + }); + + test('does not re-read from ifAbsent', () async { + expect( + await cache.readAsBytes(txt1, ifAbsent: () async => txt1Bytes), + txt1Bytes, + ); + expect( + await cache.readAsBytes(txt1, ifAbsent: () async => txt2Bytes), + txt1Bytes /* cached value */, + ); + }); + + test('can be invalidated with invalidate', () async { + expect( + await cache.readAsBytes(txt1, ifAbsent: () async => txt1Bytes), + txt1Bytes, + ); + await cache.invalidate([txt1]); + expect( + await cache.readAsBytes(txt1, ifAbsent: () async => txt2Bytes), + txt2Bytes /* updated value */, + ); + }); + }); + + group('readAsString', () { + test('reads from isAbsent', () async { + expect(await cache.readAsString(txt1, ifAbsent: () async => '1'), '1'); + }); + + test('does not re-read from isAbsent', () async { + expect(await cache.readAsString(txt1, ifAbsent: () async => '1'), '1'); + expect( + await cache.readAsString(txt1, ifAbsent: () async => '2'), + '1' /* cached value */, + ); + }); + + test('can be invalidated with invalidate', () async { + expect(await cache.readAsString(txt1, ifAbsent: () async => '1'), '1'); + await cache.invalidate([txt1]); + expect( + await cache.readAsString(txt1, ifAbsent: () async => '2'), + '2' /* updated value */, + ); + }); + }); +} diff --git a/build_runner_core/test/asset/lru_cache_test.dart b/build/test/state/lru_cache_test.dart similarity index 98% rename from build_runner_core/test/asset/lru_cache_test.dart rename to build/test/state/lru_cache_test.dart index 399beb69d..f12002217 100644 --- a/build_runner_core/test/asset/lru_cache_test.dart +++ b/build/test/state/lru_cache_test.dart @@ -4,7 +4,7 @@ @Timeout(Duration(seconds: 10)) library; -import 'package:build_runner_core/src/asset/lru_cache.dart'; +import 'package:build/src/state/lru_cache.dart'; import 'package:test/test.dart'; void main() { diff --git a/build_runner/CHANGELOG.md b/build_runner/CHANGELOG.md index b90ffaf03..9207ca551 100644 --- a/build_runner/CHANGELOG.md +++ b/build_runner/CHANGELOG.md @@ -5,6 +5,7 @@ - Start using `package:build/src/internal.dart'. - Refactor `MultiPackageAssetReader` to internal `AssetFinder`. - `FinalizedReader` no longer implements `AssetReader`. +- Refactor `CachingAssetReader` to `FilesystemCache`. ## 2.4.15 diff --git a/build_runner/lib/src/daemon/daemon_builder.dart b/build_runner/lib/src/daemon/daemon_builder.dart index 44e770ed2..8254d19f1 100644 --- a/build_runner/lib/src/daemon/daemon_builder.dart +++ b/build_runner/lib/src/daemon/daemon_builder.dart @@ -11,6 +11,8 @@ import 'package:build_daemon/daemon_builder.dart'; import 'package:build_daemon/data/build_status.dart'; import 'package:build_daemon/data/build_target.dart' hide OutputLocation; import 'package:build_daemon/data/server_log.dart'; +import 'package:build_runner_core/build_runner_core.dart' + hide BuildResult, BuildStatus; import 'package:build_runner_core/build_runner_core.dart' as core show BuildStatus; diff --git a/build_runner/lib/src/watcher/change_filter.dart b/build_runner/lib/src/watcher/change_filter.dart index 4908204ed..dc8bf0a1d 100644 --- a/build_runner/lib/src/watcher/change_filter.dart +++ b/build_runner/lib/src/watcher/change_filter.dart @@ -4,6 +4,8 @@ import 'dart:async'; import 'package:build/build.dart'; +// ignore: implementation_imports +import 'package:build/src/internal.dart'; import 'package:build_runner_core/build_runner_core.dart'; // ignore: implementation_imports import 'package:build_runner_core/src/asset_graph/graph.dart'; @@ -29,6 +31,7 @@ FutureOr shouldProcess( if (_isAddOrEditOnGeneratedFile(node, change.type)) return false; if (change.type == ChangeType.MODIFY) { // Was it really modified or just touched? + reader.filesystem.cache.invalidate([change.id]); return reader .digest(change.id) .then((newDigest) => node.lastKnownDigest != newDigest); diff --git a/build_runner/test/server/asset_handler_test.dart b/build_runner/test/server/asset_handler_test.dart index 9f4a4fb16..38c1af21d 100644 --- a/build_runner/test/server/asset_handler_test.dart +++ b/build_runner/test/server/asset_handler_test.dart @@ -13,7 +13,6 @@ import 'package:build_runner_core/src/asset_graph/node.dart'; import 'package:build_runner_core/src/generate/options.dart'; import 'package:build_runner_core/src/package_graph/target_graph.dart'; import 'package:shelf/shelf.dart'; -import 'package:test/fake.dart'; import 'package:test/test.dart'; void main() { @@ -28,7 +27,7 @@ void main() { {}, {}, buildPackageGraph({rootPackage('a'): []}), - FakeAssetReader(), + InMemoryAssetReaderWriter(), ); delegate = InMemoryRunnerAssetReaderWriter(); final packageGraph = buildPackageGraph({rootPackage('a'): []}); @@ -157,5 +156,3 @@ void main() { expect(await response.readAsString(), ''); }); } - -class FakeAssetReader with Fake implements AssetReader {} diff --git a/build_runner_core/CHANGELOG.md b/build_runner_core/CHANGELOG.md index 87f755139..c9e4de389 100644 --- a/build_runner_core/CHANGELOG.md +++ b/build_runner_core/CHANGELOG.md @@ -9,6 +9,7 @@ - `FinalizedReader` no longer implements `AssetReader`. - Add internal `Filesystem` that backs `AssetReader` and `AssetWriter` implementations. +- Refactor `CachingAssetReader` to `FilesystemCache`. ## 8.0.0 diff --git a/build_runner_core/lib/src/asset/batch.dart b/build_runner_core/lib/src/asset/batch.dart index a3ea234d3..2c0b5dfbc 100644 --- a/build_runner_core/lib/src/asset/batch.dart +++ b/build_runner_core/lib/src/asset/batch.dart @@ -80,6 +80,10 @@ final class BatchReader extends AssetReader implements AssetReaderState { BatchReader(this._inner, this._batch); + @override + BatchReader copyWith({FilesystemCache? cache}) => + BatchReader(_inner.copyWith(cache: cache), _batch); + @override Filesystem get filesystem => _inner.filesystem; diff --git a/build_runner_core/lib/src/asset/build_cache.dart b/build_runner_core/lib/src/asset/build_cache.dart index 74d9f4a1e..42d699469 100644 --- a/build_runner_core/lib/src/asset/build_cache.dart +++ b/build_runner_core/lib/src/asset/build_cache.dart @@ -37,6 +37,13 @@ class BuildCacheReader implements AssetReader, AssetReaderState { overlay: (id) => _cacheLocation(id, assetGraph, rootPackage), ); + @override + BuildCacheReader copyWith({FilesystemCache? cache}) => BuildCacheReader( + _delegate.copyWith(cache: cache), + _assetGraph, + _rootPackage, + ); + @override Filesystem get filesystem => _delegate.filesystem; diff --git a/build_runner_core/lib/src/asset/cache.dart b/build_runner_core/lib/src/asset/cache.dart deleted file mode 100644 index f38c6cbf3..000000000 --- a/build_runner_core/lib/src/asset/cache.dart +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file -// for details. All rights reserved. Use of this source code is governed by a -// BSD-style license that can be found in the LICENSE file. - -import 'dart:async'; -import 'dart:convert'; -import 'dart:typed_data'; - -import 'package:build/build.dart'; -// ignore: implementation_imports -import 'package:build/src/internal.dart'; -import 'package:crypto/crypto.dart'; -import 'package:glob/glob.dart'; - -import 'lru_cache.dart'; - -/// An [AssetReader] that caches all results from the delegate. -/// -/// Assets are cached until [invalidate] is invoked. -class CachingAssetReader implements AssetReader, AssetReaderState { - /// Cached results of [readAsBytes]. - final _bytesContentCache = LruCache>( - 1024 * 1024, - 1024 * 1024 * 512, - (value) => value is Uint8List ? value.lengthInBytes : value.length * 8, - ); - - /// Pending [readAsBytes] operations. - final _pendingBytesContentCache = >>{}; - - /// Cached results of [canRead]. - /// - /// Don't bother using an LRU cache for this since it's just booleans. - final _canReadCache = >{}; - - /// Cached results of [readAsString]. - /// - /// These are computed and stored lazily using [readAsBytes]. - /// - /// Only files read with [utf8] encoding (the default) will ever be cached. - final _stringContentCache = LruCache( - 1024 * 1024, - 1024 * 1024 * 512, - (value) => value.length, - ); - - /// Pending `readAsString` operations. - final _pendingStringContentCache = >{}; - - final AssetReader _delegate; - - CachingAssetReader(this._delegate); - - @override - Filesystem get filesystem => _delegate.filesystem; - - @override - AssetPathProvider? get assetPathProvider => _delegate.assetPathProvider; - - @override - InputTracker? get inputTracker => _delegate.inputTracker; - - @override - AssetFinder get assetFinder => _delegate.assetFinder; - - @override - Future canRead(AssetId id) => - _canReadCache.putIfAbsent(id, () => _delegate.canRead(id)); - - @override - Future digest(AssetId id) => _delegate.digest(id); - - // This is only for generators, so only `BuildStep` needs to implement it. - @override - Stream findAssets(Glob glob) => throw UnimplementedError(); - - @override - Future> readAsBytes(AssetId id, {bool cache = true}) { - var cached = _bytesContentCache[id]; - if (cached != null) return Future.value(cached); - - return _pendingBytesContentCache.putIfAbsent( - id, - () => _delegate.readAsBytes(id).then((result) { - if (cache) _bytesContentCache[id] = result; - _pendingBytesContentCache.remove(id); - return result; - }), - ); - } - - @override - Future readAsString(AssetId id, {Encoding encoding = utf8}) { - if (encoding != utf8) { - // Fallback case, we never cache the String value for the non-default, - // encoding but we do allow it to cache the bytes. - return readAsBytes(id).then(encoding.decode); - } - - var cached = _stringContentCache[id]; - if (cached != null) return Future.value(cached); - - return _pendingStringContentCache.putIfAbsent( - id, - () => readAsBytes(id, cache: false).then((bytes) { - var decoded = encoding.decode(bytes); - _stringContentCache[id] = decoded; - _pendingStringContentCache.remove(id); - return decoded; - }), - ); - } - - /// Clears all [ids] from all caches. - void invalidate(Iterable ids) { - for (var id in ids) { - _bytesContentCache.remove(id); - _canReadCache.remove(id); - _stringContentCache.remove(id); - - _pendingBytesContentCache.remove(id); - _pendingStringContentCache.remove(id); - } - } -} diff --git a/build_runner_core/lib/src/asset/file_based.dart b/build_runner_core/lib/src/asset/file_based.dart index eb8ce9a85..5b876564d 100644 --- a/build_runner_core/lib/src/asset/file_based.dart +++ b/build_runner_core/lib/src/asset/file_based.dart @@ -25,8 +25,15 @@ class FileBasedAssetReader extends AssetReader implements AssetReaderState { final PackageGraph packageGraph; - FileBasedAssetReader(this.packageGraph) - : filesystem = IoFilesystem(assetPathProvider: packageGraph); + FileBasedAssetReader(this.packageGraph, {Filesystem? filesystem}) + : filesystem = filesystem ?? IoFilesystem(assetPathProvider: packageGraph); + + @override + FileBasedAssetReader copyWith({FilesystemCache? cache}) => + FileBasedAssetReader( + packageGraph, + filesystem: filesystem.copyWith(cache: cache), + ); @override AssetPathProvider? get assetPathProvider => packageGraph; diff --git a/build_runner_core/lib/src/asset/reader.dart b/build_runner_core/lib/src/asset/reader.dart index 36160a7bb..7f98cc417 100644 --- a/build_runner_core/lib/src/asset/reader.dart +++ b/build_runner_core/lib/src/asset/reader.dart @@ -63,7 +63,7 @@ typedef CheckInvalidInput = void Function(AssetId id); /// /// Tracks the assets and globs read during this step for input dependency /// tracking. -class SingleStepReader implements AssetReader, AssetReaderState { +class SingleStepReader extends AssetReader implements AssetReaderState { @override late final AssetFinder assetFinder = FunctionAssetFinder(_findAssets); @@ -91,6 +91,18 @@ class SingleStepReader implements AssetReader, AssetReaderState { this._writtenAssets, ]); + @override + SingleStepReader copyWith({FilesystemCache? cache}) => SingleStepReader( + _delegate.copyWith(cache: cache), + _assetGraph, + _phaseNumber, + _primaryPackage, + _isReadableNode, + _checkInvalidInput, + _getGlobNode, + _writtenAssets, + ); + @override Filesystem get filesystem => _delegate.filesystem; diff --git a/build_runner_core/lib/src/asset_graph/graph.dart b/build_runner_core/lib/src/asset_graph/graph.dart index 3a1e5a52e..f6ddbbaf7 100644 --- a/build_runner_core/lib/src/asset_graph/graph.dart +++ b/build_runner_core/lib/src/asset_graph/graph.dart @@ -9,6 +9,8 @@ import 'dart:io'; import 'package:build/build.dart'; import 'package:build/experiments.dart' as experiments_zone; +// ignore: implementation_imports +import 'package:build/src/internal.dart'; import 'package:convert/convert.dart'; import 'package:crypto/crypto.dart'; import 'package:glob/glob.dart'; @@ -188,6 +190,7 @@ class AssetGraph { Iterable nodes, AssetReader digestReader, ) async { + await digestReader.filesystem.cache.invalidate(nodes.map((n) => n.id)); await Future.wait( nodes.map((node) async { node.lastKnownDigest = await digestReader.digest(node.id); diff --git a/build_runner_core/lib/src/generate/build_definition.dart b/build_runner_core/lib/src/generate/build_definition.dart index 982ea1f7b..aa081ba37 100644 --- a/build_runner_core/lib/src/generate/build_definition.dart +++ b/build_runner_core/lib/src/generate/build_definition.dart @@ -168,6 +168,7 @@ class AssetTracker { var node = assetGraph.get(id)!; var originalDigest = node.lastKnownDigest; if (originalDigest == null) return; + await _reader.filesystem.cache.invalidate([id]); var currentDigest = await _reader.digest(id); if (currentDigest != originalDigest) { updates[id] = ChangeType.MODIFY; diff --git a/build_runner_core/lib/src/generate/build_impl.dart b/build_runner_core/lib/src/generate/build_impl.dart index c2f82bbb9..ee9e98948 100644 --- a/build_runner_core/lib/src/generate/build_impl.dart +++ b/build_runner_core/lib/src/generate/build_impl.dart @@ -17,7 +17,6 @@ import 'package:path/path.dart' as p; import 'package:pool/pool.dart'; import 'package:watcher/watcher.dart'; -import '../asset/cache.dart'; import '../asset/finalized_reader.dart'; import '../asset/reader.dart'; import '../asset/writer.dart'; @@ -83,8 +82,12 @@ class BuildImpl { _targetGraph = buildDefinition.targetGraph, _reader = options.enableLowResourcesMode - ? buildDefinition.reader - : CachingAssetReader(buildDefinition.reader), + ? buildDefinition.reader.copyWith( + cache: const PassthroughFilesystemCache(), + ) + : buildDefinition.reader.copyWith( + cache: InMemoryFilesystemCache(), + ), _resolvers = options.resolvers, _writer = buildDefinition.writer, assetGraph = buildDefinition.assetGraph, @@ -293,9 +296,7 @@ class _SingleBuild { _delete, _reader, ); - if (_reader is CachingAssetReader) { - _reader.invalidate(invalidated); - } + await _reader.filesystem.cache.invalidate(invalidated); }); } @@ -976,7 +977,10 @@ class _SingleBuild { combine(md5.convert(id.toString().codeUnits).bytes as Uint8List); return; } else { - node.lastKnownDigest ??= await reader.digest(id); + if (node.lastKnownDigest == null) { + await reader.filesystem.cache.invalidate([id]); + node.lastKnownDigest = await reader.digest(id); + } } combine(node.lastKnownDigest!.bytes as Uint8List); }), diff --git a/build_runner_core/test/asset/cache_test.dart b/build_runner_core/test/asset/cache_test.dart deleted file mode 100644 index b8c3a091f..000000000 --- a/build_runner_core/test/asset/cache_test.dart +++ /dev/null @@ -1,129 +0,0 @@ -// Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file -// for details. All rights reserved. Use of this source code is governed by a -// BSD-style license that can be found in the LICENSE file. - -import 'package:_test_common/common.dart'; -import 'package:build/build.dart'; -import 'package:build_runner_core/src/asset/cache.dart'; -import 'package:test/test.dart'; - -void main() { - var fooTxt = AssetId('a', 'foo.txt'); - var missingTxt = AssetId('a', 'missing.txt'); - var fooContent = 'bar'; - var fooutf8Bytes = decodedMatches('bar'); - late InMemoryRunnerAssetReaderWriter delegate; - late CachingAssetReader reader; - - setUp(() { - delegate = - InMemoryRunnerAssetReaderWriter() - ..filesystem.writeAsStringSync(fooTxt, 'bar'); - reader = CachingAssetReader(delegate); - }); - - group('canRead', () { - test('should read from the delegate', () async { - expect(await reader.canRead(fooTxt), isTrue); - expect(await reader.canRead(missingTxt), isFalse); - expect(delegate.inputTracker.assetsRead, {fooTxt, missingTxt}); - }); - - test('should not re-read from the delegate', () async { - expect(await reader.canRead(fooTxt), isTrue); - delegate.inputTracker.assetsRead.clear(); - expect(await reader.canRead(fooTxt), isTrue); - expect(delegate.inputTracker.assetsRead, isEmpty); - }); - - test('can be invalidated with invalidate', () async { - expect(await reader.canRead(fooTxt), isTrue); - delegate.inputTracker.assetsRead.clear(); - expect(delegate.inputTracker.assetsRead, isEmpty); - - reader.invalidate([fooTxt]); - expect(await reader.canRead(fooTxt), isTrue); - expect(delegate.inputTracker.assetsRead, [fooTxt]); - }); - }); - - group('readAsBytes', () { - test('should read from the delegate', () async { - expect(await reader.readAsBytes(fooTxt), fooutf8Bytes); - expect(delegate.inputTracker.assetsRead, [fooTxt]); - }); - - test('should not re-read from the delegate', () async { - expect(await reader.readAsBytes(fooTxt), fooutf8Bytes); - delegate.inputTracker.assetsRead.clear(); - expect(await reader.readAsBytes(fooTxt), fooutf8Bytes); - expect(delegate.inputTracker.assetsRead, isEmpty); - }); - - test('can be invalidated with invalidate', () async { - expect(await reader.readAsBytes(fooTxt), fooutf8Bytes); - delegate.inputTracker.assetsRead.clear(); - expect(delegate.inputTracker.assetsRead, isEmpty); - - reader.invalidate([fooTxt]); - expect(await reader.readAsBytes(fooTxt), fooutf8Bytes); - expect(delegate.inputTracker.assetsRead, [fooTxt]); - }); - - test('should not cache bytes during readAsString calls', () async { - expect(await reader.readAsString(fooTxt), fooContent); - expect(delegate.inputTracker.assetsRead, [fooTxt]); - delegate.inputTracker.assetsRead.clear(); - - expect(await reader.readAsBytes(fooTxt), fooutf8Bytes); - expect(delegate.inputTracker.assetsRead, [fooTxt]); - }); - }); - - group('readAsString', () { - test('should read from the delegate', () async { - expect(await reader.readAsString(fooTxt), fooContent); - expect(delegate.inputTracker.assetsRead, [fooTxt]); - }); - - test('should not re-read from the delegate', () async { - expect(await reader.readAsString(fooTxt), fooContent); - delegate.inputTracker.assetsRead.clear(); - expect(await reader.readAsString(fooTxt), fooContent); - expect(delegate.inputTracker.assetsRead, isEmpty); - }); - - test('can be invalidated with invalidate', () async { - expect(await reader.readAsString(fooTxt), fooContent); - delegate.inputTracker.assetsRead.clear(); - expect(delegate.inputTracker.assetsRead, isEmpty); - - reader.invalidate([fooTxt]); - expect(await reader.readAsString(fooTxt), fooContent); - expect(delegate.inputTracker.assetsRead, [fooTxt]); - }); - - test('uses cached bytes if available', () async { - expect(await reader.readAsBytes(fooTxt), fooutf8Bytes); - expect(delegate.inputTracker.assetsRead, [fooTxt]); - delegate.inputTracker.assetsRead.clear(); - - expect(await reader.readAsString(fooTxt), fooContent); - expect(delegate.inputTracker.assetsRead, isEmpty); - }); - }); - - group('digest', () { - test('should read from the delegate', () async { - expect(await reader.digest(fooTxt), isNotNull); - expect(delegate.inputTracker.assetsRead, [fooTxt]); - }); - - test('should re-read from the delegate (no cache)', () async { - expect(await reader.digest(fooTxt), isNotNull); - delegate.inputTracker.assetsRead.clear(); - expect(await reader.digest(fooTxt), isNotNull); - expect(delegate.inputTracker.assetsRead, [fooTxt]); - }); - }); -} diff --git a/build_runner_core/test/asset/finalized_reader_test.dart b/build_runner_core/test/asset/finalized_reader_test.dart index dec755c87..ee206c7cc 100644 --- a/build_runner_core/test/asset/finalized_reader_test.dart +++ b/build_runner_core/test/asset/finalized_reader_test.dart @@ -14,7 +14,6 @@ import 'package:build_runner_core/src/generate/options.dart'; import 'package:build_runner_core/src/generate/phase.dart'; import 'package:build_runner_core/src/package_graph/target_graph.dart'; import 'package:glob/glob.dart'; -import 'package:test/fake.dart'; import 'package:test/test.dart'; void main() { @@ -35,7 +34,7 @@ void main() { {}, {}, packageGraph, - _FakeAssetReader(), + InMemoryAssetReaderWriter(), ); }); @@ -99,5 +98,3 @@ void main() { }); }); } - -class _FakeAssetReader with Fake implements AssetReader {} diff --git a/build_runner_core/test/asset_graph/graph_test.dart b/build_runner_core/test/asset_graph/graph_test.dart index cf5f5da18..41c746f70 100644 --- a/build_runner_core/test/asset_graph/graph_test.dart +++ b/build_runner_core/test/asset_graph/graph_test.dart @@ -18,9 +18,13 @@ import 'package:test/test.dart'; import 'package:watcher/watcher.dart'; void main() { - final digestReader = StubAssetReader(); + late InMemoryAssetReaderWriter digestReader; final fooPackageGraph = buildPackageGraph({rootPackage('foo'): []}); + setUp(() async { + digestReader = InMemoryAssetReaderWriter(); + }); + group('AssetGraph', () { late AssetGraph graph; @@ -223,6 +227,9 @@ void main() { ); setUp(() async { + for (final id in [primaryInputId, internalId]) { + digestReader.filesystem.writeAsStringSync(id, 'contents of $id'); + } graph = await AssetGraph.build( buildPhases, {primaryInputId, excludedInputId}, @@ -457,6 +464,10 @@ void main() { graph.add(globNode); var coolAssetId = AssetId('foo', 'lib/really.cool'); + digestReader.filesystem.writeAsStringSync( + coolAssetId, + 'contents of $coolAssetId', + ); Future checkChangeType(ChangeType changeType) async { var changes = {coolAssetId: changeType}; @@ -525,6 +536,20 @@ void main() { group('regression tests', () { test('build can chains of pre-existing to-source outputs', () async { + final sources = { + makeAssetId('foo|lib/1.txt'), + makeAssetId('foo|lib/2.txt'), + // All the following are actually old outputs. + makeAssetId('foo|lib/1.a.txt'), + makeAssetId('foo|lib/1.a.b.txt'), + makeAssetId('foo|lib/2.a.txt'), + makeAssetId('foo|lib/2.a.b.txt'), + makeAssetId('foo|lib/2.a.b.c.txt'), + }; + + for (final id in sources) { + digestReader.filesystem.writeAsStringSync(id, 'contents of $id'); + } final graph = await AssetGraph.build( [ InBuildPhase( @@ -545,16 +570,7 @@ void main() { hideOutput: false, ), ], - { - makeAssetId('foo|lib/1.txt'), - makeAssetId('foo|lib/2.txt'), - // All the following are actually old outputs. - makeAssetId('foo|lib/1.a.txt'), - makeAssetId('foo|lib/1.a.b.txt'), - makeAssetId('foo|lib/2.a.txt'), - makeAssetId('foo|lib/2.a.b.txt'), - makeAssetId('foo|lib/2.a.b.c.txt'), - }, + sources, {}, fooPackageGraph, digestReader, @@ -583,6 +599,10 @@ void main() { test('allows running on generated inputs that do not match target ' 'source globs', () async { + final sources = {makeAssetId('foo|lib/1.txt')}; + for (final id in sources) { + digestReader.filesystem.writeAsStringSync(id, 'contents of $id'); + } final graph = await AssetGraph.build( [ InBuildPhase( @@ -595,7 +615,7 @@ void main() { targetSources: const InputSet(include: ['lib/*.txt']), ), ], - {makeAssetId('foo|lib/1.txt')}, + sources, {}, fooPackageGraph, digestReader, @@ -628,9 +648,13 @@ void main() { 'foo', ), ]; + final sources = {makeAssetId('foo|lib/b.anchor')}; + for (final id in sources) { + digestReader.filesystem.writeAsStringSync(id, 'contents of $id'); + } final graph = await AssetGraph.build( buildPhases, - {makeAssetId('foo|lib/b.anchor')}, + sources, {}, fooPackageGraph, digestReader, @@ -662,6 +686,10 @@ void main() { test('https://github.com/dart-lang/build/issues/1804', () async { final source = AssetId('a', 'lib/a.dart'); + digestReader.filesystem.writeAsStringSync( + source, + 'contents of $source', + ); final renamedSource = AssetId('a', 'lib/A.dart'); final generatedDart = AssetId('a', 'lib/a.g.dart'); final generatedPart = AssetId('a', 'lib/a.g.part'); diff --git a/build_test/CHANGELOG.md b/build_test/CHANGELOG.md index 99a35e31e..1d4037ba4 100644 --- a/build_test/CHANGELOG.md +++ b/build_test/CHANGELOG.md @@ -23,6 +23,8 @@ instead. - Breaking change: merged `InMemoryAssetReader` and `InMemoryAssetWriter` into `InMemoryAssetReaderWriter`. +- Refactor `CachingAssetReader` to `FilesystemCache`. +- Remove `StubAssetReader`. Use `InMemoryAssetReaderWriter` instead. ## 2.2.3 diff --git a/build_test/lib/build_test.dart b/build_test/lib/build_test.dart index 8d43da607..1768cfb38 100644 --- a/build_test/lib/build_test.dart +++ b/build_test/lib/build_test.dart @@ -14,7 +14,6 @@ export 'src/package_reader.dart' show PackageAssetReader; export 'src/record_logs.dart'; export 'src/resolve_source.dart' show resolveAsset, resolveSource, resolveSources, useAssetReader; -export 'src/stub_reader.dart'; export 'src/stub_writer.dart'; export 'src/test_builder.dart'; export 'src/written_asset_reader.dart'; diff --git a/build_test/lib/src/in_memory_reader_writer.dart b/build_test/lib/src/in_memory_reader_writer.dart index f874d5bd1..0e09c3960 100644 --- a/build_test/lib/src/in_memory_reader_writer.dart +++ b/build_test/lib/src/in_memory_reader_writer.dart @@ -6,8 +6,6 @@ import 'dart:convert'; import 'package:build/build.dart'; // ignore: implementation_imports import 'package:build/src/internal.dart'; -import 'package:convert/convert.dart'; -import 'package:crypto/crypto.dart'; import 'package:glob/glob.dart'; /// An [AssetReader] that records which assets have been read to [assetsRead]. @@ -19,12 +17,12 @@ abstract class RecordingAssetReader implements AssetReader { /// assets. /// /// TODO(davidmorgan): merge into `FileBasedReader` and `FileBasedWriter`. -class InMemoryAssetReaderWriter - implements AssetReader, AssetReaderState, AssetWriter { +class InMemoryAssetReaderWriter extends AssetReader + implements AssetReaderState, AssetWriter { @override late final AssetFinder assetFinder = FunctionAssetFinder(_findAssets); - final InMemoryFilesystem _filesystem = InMemoryFilesystem(); + final InMemoryFilesystem _filesystem; final String? rootPackage; @@ -34,7 +32,15 @@ class InMemoryAssetReaderWriter /// Create a new asset reader/writer. /// /// May optionally define a [rootPackage], which is required for some APIs. - InMemoryAssetReaderWriter({this.rootPackage}); + InMemoryAssetReaderWriter({this.rootPackage, InMemoryFilesystem? filesystem}) + : _filesystem = filesystem ?? InMemoryFilesystem(); + + @override + InMemoryAssetReaderWriter copyWith({FilesystemCache? cache}) => + InMemoryAssetReaderWriter( + rootPackage: rootPackage, + filesystem: _filesystem.copyWith(cache: cache), + ); @override AssetPathProvider? get assetPathProvider => null; @@ -91,14 +97,4 @@ class InMemoryAssetReaderWriter String contents, { Encoding encoding = utf8, }) async => filesystem.writeAsString(id, contents, encoding: encoding); - - @override - Future digest(AssetId id) async { - var digestSink = AccumulatorSink(); - md5.startChunkedConversion(digestSink) - ..add(await readAsBytes(id)) - ..add(id.toString().codeUnits) - ..close(); - return digestSink.events.first; - } } diff --git a/build_test/lib/src/stub_reader.dart b/build_test/lib/src/stub_reader.dart deleted file mode 100644 index 3c3711909..000000000 --- a/build_test/lib/src/stub_reader.dart +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file -// for details. All rights reserved. Use of this source code is governed by a -// BSD-style license that can be found in the LICENSE file. -import 'dart:convert'; - -import 'package:build/build.dart'; -import 'package:crypto/crypto.dart'; -import 'package:glob/glob.dart'; - -/// A no-op implementation of [AssetReader]. -class StubAssetReader extends AssetReader { - StubAssetReader(); - - @override - Future canRead(AssetId id) => Future.value(false); - - @override - Future> readAsBytes(AssetId id) => Future.value([]); - - @override - Future readAsString(AssetId id, {Encoding encoding = utf8}) => - Future.value(''); - - // This is only for generators, so only `BuildStep` needs to implement it. - @override - Stream findAssets(Glob glob) => throw UnimplementedError(); - - @override - Future digest(AssetId id) => Future.value(Digest([1, 2, 3])); -} diff --git a/build_test/lib/src/written_asset_reader.dart b/build_test/lib/src/written_asset_reader.dart index 25be8874e..4153045c8 100644 --- a/build_test/lib/src/written_asset_reader.dart +++ b/build_test/lib/src/written_asset_reader.dart @@ -26,6 +26,10 @@ class WrittenAssetReader extends AssetReader implements AssetReaderState { WrittenAssetReader(this.source, [this.filterSpy]); + @override + WrittenAssetReader copyWith({FilesystemCache? cache}) => + WrittenAssetReader(source.copyWith(cache: cache), filterSpy); + @override Filesystem get filesystem => source.filesystem;