Skip to content

Use sync I/O. #4000

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
May 8, 2025
Merged

Use sync I/O. #4000

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
119 changes: 5 additions & 114 deletions build/lib/src/state/filesystem.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,36 +6,20 @@ import 'dart:convert';
import 'dart:io';
import 'dart:typed_data';

import 'package:pool/pool.dart';

/// The filesystem the build is running on.
///
/// Methods behave as the `dart:io` methods with the same names, with some
/// exceptions noted in the docs.
abstract interface class Filesystem {
/// Whether the file exists.
Future<bool> exists(String path);

/// Whether the file exists.
bool existsSync(String path);

/// Reads a file as a string.
Future<String> readAsString(String path, {Encoding encoding = utf8});

/// Reads a file as a string.
String readAsStringSync(String path, {Encoding encoding = utf8});

/// Reads a file as bytes.
Future<Uint8List> readAsBytes(String path);

/// Reads a file as bytes.
Uint8List readAsBytesSync(String path);

/// Deletes a file.
///
/// If the file does not exist, does nothing.
Future<void> delete(String path);

/// Deletes a file.
///
/// If the file does not exist, does nothing.
Expand All @@ -44,7 +28,7 @@ abstract interface class Filesystem {
/// Deletes a directory recursively.
///
/// If the directory does not exist, does nothing.
Future<void> deleteDirectory(String path);
void deleteDirectorySync(String path);

/// Writes a file.
///
Expand All @@ -55,48 +39,20 @@ abstract interface class Filesystem {
Encoding encoding = utf8,
});

/// Writes a file.
///
/// Creates enclosing directories as needed if they don't exist.
Future<void> writeAsString(
String path,
String contents, {
Encoding encoding = utf8,
});

/// Writes a file.
///
/// Creates enclosing directories as needed if they don't exist.
void writeAsBytesSync(String path, List<int> contents);

/// Writes a file.
///
/// Creates enclosing directories as needed if they don't exist.
Future<void> writeAsBytes(String path, List<int> contents);
}

/// The `dart:io` filesystem.
class IoFilesystem implements Filesystem {
/// Pool for async file operations.
final _pool = Pool(32);

@override
Future<bool> exists(String path) => _pool.withResource(File(path).exists);

@override
bool existsSync(String path) => File(path).existsSync();

@override
Future<Uint8List> readAsBytes(String path) =>
_pool.withResource(File(path).readAsBytes);

@override
Uint8List readAsBytesSync(String path) => File(path).readAsBytesSync();

@override
Future<String> readAsString(String path, {Encoding encoding = utf8}) =>
_pool.withResource(() => File(path).readAsString(encoding: encoding));

@override
String readAsStringSync(String path, {Encoding encoding = utf8}) =>
File(path).readAsStringSync(encoding: encoding);
Expand All @@ -108,19 +64,9 @@ class IoFilesystem implements Filesystem {
}

@override
Future<void> delete(String path) {
return _pool.withResource(() async {
final file = File(path);
if (await file.exists()) await file.delete();
});
}

@override
Future<void> deleteDirectory(String path) {
return _pool.withResource(() async {
final directory = Directory(path);
if (await directory.exists()) await directory.delete(recursive: true);
});
void deleteDirectorySync(String path) {
final directory = Directory(path);
if (directory.existsSync()) directory.deleteSync(recursive: true);
}

@override
Expand All @@ -134,15 +80,6 @@ class IoFilesystem implements Filesystem {
file.writeAsBytesSync(contents);
}

@override
Future<void> writeAsBytes(String path, List<int> contents) {
return _pool.withResource(() async {
final file = File(path);
await file.parent.create(recursive: true);
await file.writeAsBytes(contents);
});
}

@override
void writeAsStringSync(
String path,
Expand All @@ -153,19 +90,6 @@ class IoFilesystem implements Filesystem {
file.parent.createSync(recursive: true);
file.writeAsStringSync(contents, encoding: encoding);
}

@override
Future<void> writeAsString(
String path,
String contents, {
Encoding encoding = utf8,
}) {
return _pool.withResource(() async {
final file = File(path);
await file.parent.create(recursive: true);
await file.writeAsString(contents, encoding: encoding);
});
}
}

/// An in-memory [Filesystem].
Expand All @@ -175,53 +99,30 @@ class InMemoryFilesystem implements Filesystem {
/// The paths to all files present on the filesystem.
Iterable<String> get filePaths => _files.keys;

@override
Future<bool> exists(String path) => Future.value(_files.containsKey(path));

@override
bool existsSync(String path) => _files.containsKey(path);

@override
Future<Uint8List> readAsBytes(String path) => Future.value(_files[path]!);

@override
Uint8List readAsBytesSync(String path) => _files[path]!;

@override
Future<String> readAsString(String path, {Encoding encoding = utf8}) =>
Future.value(encoding.decode(_files[path]!));

@override
String readAsStringSync(String path, {Encoding encoding = utf8}) =>
encoding.decode(_files[path]!);

@override
Future<void> delete(String path) {
_files.remove(path);
return Future.value();
}

@override
void deleteSync(String path) => _files.remove(path);

@override
Future<void> deleteDirectory(String path) {
void deleteDirectorySync(String path) {
final prefix = '$path/';
_files.removeWhere((filePath, _) => filePath.startsWith(prefix));
return Future.value();
}

@override
void writeAsBytesSync(String path, List<int> contents) {
_files[path] = Uint8List.fromList(contents);
}

@override
Future<void> writeAsBytes(String path, List<int> contents) {
_files[path] = Uint8List.fromList(contents);
return Future.value();
}

@override
void writeAsStringSync(
String path,
Expand All @@ -230,14 +131,4 @@ class InMemoryFilesystem implements Filesystem {
}) {
_files[path] = Uint8List.fromList(encoding.encode(contents));
}

@override
Future<void> writeAsString(
String path,
String contents, {
Encoding encoding = utf8,
}) {
_files[path] = Uint8List.fromList(encoding.encode(contents));
return Future.value();
}
}
99 changes: 36 additions & 63 deletions build/lib/src/state/filesystem_cache.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,30 +14,25 @@ import 'lru_cache.dart';
/// TODO(davidmorgan): benchmark, optimize the caching strategy.
abstract interface class FilesystemCache {
/// Clears all [ids] from all caches.
///
/// Waits for any pending reads to complete first.
Future<void> invalidate(Iterable<AssetId> ids);
void invalidate(Iterable<AssetId> ids);

/// Whether [id] exists.
///
/// Returns a cached result if available, or caches and returns `ifAbsent()`.
Future<bool> exists(AssetId id, {required Future<bool> Function() ifAbsent});
bool exists(AssetId id, {required bool Function() ifAbsent});

/// Reads [id] as bytes.
///
/// Returns a cached result if available, or caches and returns `ifAbsent()`.
Future<Uint8List> readAsBytes(
AssetId id, {
required Future<Uint8List> Function() ifAbsent,
});
Uint8List readAsBytes(AssetId id, {required Uint8List Function() ifAbsent});

/// Reads [id] as a `String`.
///
/// Returns a cached result if available, or caches and returns `ifAbsent()`.
Future<String> readAsString(
String readAsString(
AssetId id, {
Encoding encoding = utf8,
required Future<Uint8List> Function() ifAbsent,
required Uint8List Function() ifAbsent,
});
}

Expand All @@ -49,23 +44,18 @@ class PassthroughFilesystemCache implements FilesystemCache {
Future<void> invalidate(Iterable<AssetId> ids) async {}

@override
Future<bool> exists(
AssetId id, {
required Future<bool> Function() ifAbsent,
}) => ifAbsent();
bool exists(AssetId id, {required bool Function() ifAbsent}) => ifAbsent();

@override
Future<Uint8List> readAsBytes(
AssetId id, {
required Future<Uint8List> Function() ifAbsent,
}) => ifAbsent();
Uint8List readAsBytes(AssetId id, {required Uint8List Function() ifAbsent}) =>
ifAbsent();

@override
Future<String> readAsString(
String readAsString(
AssetId id, {
Encoding encoding = utf8,
required Future<Uint8List> Function() ifAbsent,
}) async => encoding.decode(await ifAbsent());
required Uint8List Function() ifAbsent,
}) => encoding.decode(ifAbsent());
}

/// [FilesystemCache] that stores data in memory.
Expand All @@ -77,13 +67,10 @@ class InMemoryFilesystemCache implements FilesystemCache {
(value) => value.lengthInBytes,
);

/// Pending [readAsBytes] operations.
final _pendingBytesContentCache = <AssetId, Future<Uint8List>>{};

/// Cached results of [exists].
///
/// Don't bother using an LRU cache for this since it's just booleans.
final _canReadCache = <AssetId, Future<bool>>{};
final _existsCache = <AssetId, bool>{};

/// Cached results of [readAsString].
///
Expand All @@ -96,64 +83,50 @@ class InMemoryFilesystemCache implements FilesystemCache {
(value) => value.length,
);

/// Pending `readAsString` operations.
final _pendingStringContentCache = <AssetId, Future<String>>{};

@override
Future<void> invalidate(Iterable<AssetId> ids) async {
// First finish all pending operations, as they will write to the cache.
for (var id in ids) {
await _canReadCache.remove(id);
await _pendingBytesContentCache.remove(id);
await _pendingStringContentCache.remove(id);
}
for (var id in ids) {
_existsCache.remove(id);
_bytesContentCache.remove(id);
_stringContentCache.remove(id);
}
}

@override
Future<bool> exists(
AssetId id, {
required Future<bool> Function() ifAbsent,
}) => _canReadCache.putIfAbsent(id, ifAbsent);
bool exists(AssetId id, {required bool Function() ifAbsent}) =>
_existsCache.putIfAbsent(id, ifAbsent);

@override
Future<Uint8List> readAsBytes(
AssetId id, {
required Future<Uint8List> Function() ifAbsent,
}) {
var cached = _bytesContentCache[id];
if (cached != null) return Future.value(cached);

return _pendingBytesContentCache.putIfAbsent(id, () async {
final result = await ifAbsent();
_bytesContentCache[id] = result;
unawaited(_pendingBytesContentCache.remove(id));
return result;
});
Uint8List readAsBytes(AssetId id, {required Uint8List Function() ifAbsent}) {
final maybeResult = _bytesContentCache[id];
if (maybeResult != null) return maybeResult;

final result = ifAbsent();
_bytesContentCache[id] = result;
return result;
}

@override
Future<String> readAsString(
String readAsString(
AssetId id, {
Encoding encoding = utf8,
required Future<Uint8List> Function() ifAbsent,
}) async {
required Uint8List Function() ifAbsent,
}) {
if (encoding != utf8) {
final bytes = await readAsBytes(id, ifAbsent: ifAbsent);
final bytes = readAsBytes(id, ifAbsent: ifAbsent);
return encoding.decode(bytes);
}

var cached = _stringContentCache[id];
if (cached != null) return cached;
final maybeResult = _stringContentCache[id];
if (maybeResult != null) return maybeResult;

return _pendingStringContentCache.putIfAbsent(id, () async {
final bytes = await ifAbsent();
final result = _stringContentCache[id] = utf8.decode(bytes);
unawaited(_pendingStringContentCache.remove(id));
return result;
});
var bytes = _bytesContentCache[id];
if (bytes == null) {
bytes = ifAbsent();
_bytesContentCache[id] = bytes;
}
final result = utf8.decode(bytes);
_stringContentCache[id] = result;
return result;
}
}
Loading
Loading