diff --git a/pkg/pub_dartdoc/bin/pub_dartdoc.dart b/pkg/pub_dartdoc/bin/pub_dartdoc.dart index 5697d54ba7..ac6295436b 100644 --- a/pkg/pub_dartdoc/bin/pub_dartdoc.dart +++ b/pkg/pub_dartdoc/bin/pub_dartdoc.dart @@ -6,20 +6,33 @@ import 'package:dartdoc/dartdoc.dart'; import 'package:dartdoc/options.dart'; import 'package:pub_dartdoc/pub_data_generator.dart'; +import 'package:pub_dartdoc/src/pub_hooks.dart'; Future main(List arguments) async { - final config = await parseOptions(pubPackageMetaProvider, arguments); + // pub hooks + final pubResourceProvider = + PubResourceProvider(pubPackageMetaProvider.resourceProvider); + final pubMetaProvider = + PubPackageMetaProvider(pubPackageMetaProvider, pubResourceProvider); + + // dartdoc initialization + final config = await parseOptions(pubMetaProvider, arguments); if (config == null) { throw ArgumentError(); } + pubResourceProvider.setOutput(config.output); + final packageConfigProvider = PhysicalPackageConfigProvider(); final packageBuilder = - PubPackageBuilder(config, pubPackageMetaProvider, packageConfigProvider); + PubPackageBuilder(config, pubMetaProvider, packageConfigProvider); final dartdoc = config.generateDocs ? await Dartdoc.fromContext(config, packageBuilder) : await Dartdoc.withEmptyGenerator(config, packageBuilder); final results = await dartdoc.generateDocs(); - final pubDataGenerator = PubDataGenerator(config.inputDir); - await pubDataGenerator.generate(results.packageGraph, config.output); + final pubDataGenerator = + PubDataGenerator(config.inputDir, pubResourceProvider); + pubDataGenerator.generate(results.packageGraph, config.output); + + pubResourceProvider.writeFilesToDiskSync(); } diff --git a/pkg/pub_dartdoc/lib/pub_data_generator.dart b/pkg/pub_dartdoc/lib/pub_data_generator.dart index 2600e13394..9481e809b6 100644 --- a/pkg/pub_dartdoc/lib/pub_data_generator.dart +++ b/pkg/pub_dartdoc/lib/pub_data_generator.dart @@ -2,10 +2,9 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. -import 'dart:async'; import 'dart:convert' as convert; -import 'dart:io'; +import 'package:analyzer/file_system/file_system.dart'; import 'package:dartdoc/dartdoc.dart'; import 'package:path/path.dart' as p; @@ -17,11 +16,11 @@ const fileName = 'pub-data.json'; /// [PubDartdocData] instance. class PubDataGenerator { final String _inputDirectory; + final ResourceProvider _resourceProvider; - PubDataGenerator(this._inputDirectory); + PubDataGenerator(this._inputDirectory, this._resourceProvider); - Future generate( - PackageGraph packageGraph, String outputDirectoryPath) async { + void generate(PackageGraph packageGraph, String outputDirectoryPath) { final modelElements = packageGraph.allCanonicalModelElements .where((elem) => elem.isPublic) .where((elem) => p.isWithin(_inputDirectory, elem.sourceFileName)) @@ -89,8 +88,9 @@ class PubDataGenerator { PubDartdocData(coverage: coverage, apiElements: apiElements); final fileName = 'pub-data.json'; - final outputFile = File(p.join(outputDirectoryPath, fileName)); - await outputFile.writeAsString(convert.json.encode(extract.toJson())); + _resourceProvider + .getFile(p.join(outputDirectoryPath, fileName)) + .writeAsStringSync(convert.json.encode(extract.toJson())); } // Inherited member, should not show up in pub-data.json diff --git a/pkg/pub_dartdoc/lib/src/pub_hooks.dart b/pkg/pub_dartdoc/lib/src/pub_hooks.dart new file mode 100644 index 0000000000..50493dc709 --- /dev/null +++ b/pkg/pub_dartdoc/lib/src/pub_hooks.dart @@ -0,0 +1,198 @@ +// Copyright (c) 2021, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:convert'; +import 'dart:io' as io; + +import 'package:analyzer/dart/element/element.dart'; +import 'package:analyzer/file_system/file_system.dart'; +import 'package:analyzer/file_system/memory_file_system.dart'; +// ignore: implementation_imports +import 'package:analyzer/src/generated/sdk.dart'; +// ignore: implementation_imports +import 'package:analyzer/src/generated/source.dart'; +import 'package:dartdoc/dartdoc.dart'; +import 'package:path/path.dart' as p; +import 'package:watcher/watcher.dart'; + +const _maxFileCount = 10000; +const _maxTotalLengthBytes = 10 * 1024 * 1024; + +/// Creates an overlay file system with binary file support on top +/// of the input sources. +/// +/// TODO: Use a propr overlay in-memory filesystem with binary support, +/// instead of overriding file writes in the output path. +class PubResourceProvider implements ResourceProvider { + final ResourceProvider _defaultProvider; + final _memoryResourceProvider = MemoryResourceProvider(); + String _outputPath; + int _fileCount = 0; + int _totalLengthBytes = 0; + + PubResourceProvider(this._defaultProvider); + + /// Writes in-memory files to disk. + void writeFilesToDiskSync() { + void storeFolder(Folder rs) { + for (final c in rs.getChildren()) { + if (c is Folder) { + storeFolder(c); + } else if (c is File) { + final file = io.File(c.path); + file.parent.createSync(recursive: true); + file.writeAsBytes(c.readAsBytesSync()); + } + } + } + + storeFolder(_memoryResourceProvider.getFolder(_outputPath)); + } + + /// Checks if we have reached any file write limit before storing the bytes. + void _aboutToWriteBytes(int length) { + _fileCount++; + _totalLengthBytes += length; + if (_fileCount > _maxFileCount) { + throw AssertionError( + 'Reached $_maxFileCount files in the output directory.'); + } + if (_totalLengthBytes > _maxTotalLengthBytes) { + throw AssertionError( + 'Reached $_maxTotalLengthBytes bytes in the output directory.'); + } + } + + void setOutput(String value) { + _outputPath = value; + } + + bool _isOutput(String path) { + return _outputPath != null && + (path == _outputPath || p.isWithin(_outputPath, path)); + } + + ResourceProvider _rp(String path) => + _isOutput(path) ? _memoryResourceProvider : _defaultProvider; + + @override + File getFile(String path) => _File(this, _rp(path).getFile(path)); + + @override + Folder getFolder(String path) => _rp(path).getFolder(path); + + @override + Future> getModificationTimes(List sources) async { + return _defaultProvider.getModificationTimes(sources); + } + + @override + Resource getResource(String path) => _rp(path).getResource(path); + + @override + Folder getStateLocation(String pluginId) { + return _defaultProvider.getStateLocation(pluginId); + } + + @override + p.Context get pathContext => _defaultProvider.pathContext; +} + +class _File implements File { + final PubResourceProvider _provider; + final File _delegate; + _File(this._provider, this._delegate); + + @override + Stream get changes => _delegate.changes; + + @override + File copyTo(Folder parentFolder) => _delegate.copyTo(parentFolder); + + @override + Source createSource([Uri uri]) => _delegate.createSource(uri); + + @override + void delete() => _delegate.delete(); + + @override + bool get exists => _delegate.exists; + + @override + bool isOrContains(String path) => _delegate.isOrContains(path); + + @override + int get lengthSync => _delegate.lengthSync; + + @override + int get modificationStamp => _delegate.modificationStamp; + + @override + Folder get parent => _delegate.parent2; + + @override + Folder get parent2 => _delegate.parent2; + + @override + String get path => _delegate.path; + + @override + ResourceProvider get provider => _delegate.provider; + + @override + List readAsBytesSync() => _delegate.readAsBytesSync(); + + @override + String readAsStringSync() => _delegate.readAsStringSync(); + + @override + File renameSync(String newPath) => _delegate.renameSync(newPath); + + @override + Resource resolveSymbolicLinksSync() => _delegate.resolveSymbolicLinksSync(); + + @override + String get shortName => _delegate.shortName; + + @override + Uri toUri() => _delegate.toUri(); + + @override + void writeAsBytesSync(List bytes) { + _provider._aboutToWriteBytes(bytes.length); + _delegate.writeAsBytesSync(bytes); + } + + @override + void writeAsStringSync(String content) { + writeAsBytesSync(utf8.encode(content)); + } +} + +/// Allows the override of [resourceProvider]. +class PubPackageMetaProvider implements PackageMetaProvider { + final PackageMetaProvider _delegate; + final ResourceProvider _resourceProvider; + + PubPackageMetaProvider(this._delegate, this._resourceProvider); + + @override + DartSdk get defaultSdk => _delegate.defaultSdk; + + @override + Folder get defaultSdkDir => _delegate.defaultSdkDir; + + @override + PackageMeta fromDir(Folder dir) => _delegate.fromDir(dir); + + @override + PackageMeta fromElement(LibraryElement library, String s) => + _delegate.fromElement(library, s); + + @override + PackageMeta fromFilename(String s) => _delegate.fromFilename(s); + + @override + ResourceProvider get resourceProvider => _resourceProvider; +} diff --git a/pkg/pub_dartdoc/pubspec.lock b/pkg/pub_dartdoc/pubspec.lock index d7834ebcb4..23d16d956b 100644 --- a/pkg/pub_dartdoc/pubspec.lock +++ b/pkg/pub_dartdoc/pubspec.lock @@ -9,7 +9,7 @@ packages: source: hosted version: "19.0.0" analyzer: - dependency: transitive + dependency: "direct main" description: name: analyzer url: "https://pub.dartlang.org" diff --git a/pkg/pub_dartdoc/pubspec.yaml b/pkg/pub_dartdoc/pubspec.yaml index 06f12f07fc..9c1225d43d 100644 --- a/pkg/pub_dartdoc/pubspec.yaml +++ b/pkg/pub_dartdoc/pubspec.yaml @@ -4,6 +4,7 @@ description: The customized dartdoc for pub.dartlang.org. environment: sdk: ">=2.6.0 <3.0.0" dependencies: + analyzer: any # whatever dartdoc supports path: ^1.8.0 pub_dartdoc_data: path: ../pub_dartdoc_data diff --git a/pkg/pub_dartdoc/test/file_limit_test.dart b/pkg/pub_dartdoc/test/file_limit_test.dart new file mode 100644 index 0000000000..c7734a1be1 --- /dev/null +++ b/pkg/pub_dartdoc/test/file_limit_test.dart @@ -0,0 +1,30 @@ +// Copyright (c) 2021, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'package:analyzer/file_system/memory_file_system.dart'; +import 'package:test/test.dart'; + +import 'package:pub_dartdoc/src/pub_hooks.dart'; + +void main() { + test('limit number of files', () { + final provider = PubResourceProvider(MemoryResourceProvider()); + + for (var i = 0; i < 10000; i++) { + provider.getFile('/tmp/$i.txt').writeAsStringSync('x'); + } + + expect(() => provider.getFile('/tmp/next.txt').writeAsStringSync('next'), + throwsA(isA())); + }); + + test('limit total bytes', () { + final provider = PubResourceProvider(MemoryResourceProvider()); + provider + .getFile('/tmp/1') + .writeAsBytesSync(List.filled(10 * 1024 * 1024, 0)); + expect(() => provider.getFile('/tmp/2').writeAsBytesSync([0]), + throwsA(isA())); + }); +}