Limit file count and total length in pkg/pub_dartdoc.

pkg/pub_dartdoc/bin/pub_dartdoc.dart

@@ -6,20 +6,33 @@ import 'package:dartdoc/dartdoc.dart';
 import 'package:dartdoc/options.dart';
 
 import 'package:pub_dartdoc/pub_data_generator.dart';
+import 'package:pub_dartdoc/src/pub_hooks.dart';
 
 Future<void> main(List<String> arguments) async {
-  final config = await parseOptions(pubPackageMetaProvider, arguments);
+  // pub hooks
+  final pubResourceProvider =
+      PubResourceProvider(pubPackageMetaProvider.resourceProvider);
+  final pubMetaProvider =
+      PubPackageMetaProvider(pubPackageMetaProvider, pubResourceProvider);
+
+  // dartdoc initialization
+  final config = await parseOptions(pubMetaProvider, arguments);
   if (config == null) {
     throw ArgumentError();
   }
+  pubResourceProvider.setOutput(config.output);
+
   final packageConfigProvider = PhysicalPackageConfigProvider();
   final packageBuilder =
-      PubPackageBuilder(config, pubPackageMetaProvider, packageConfigProvider);
+      PubPackageBuilder(config, pubMetaProvider, packageConfigProvider);
   final dartdoc = config.generateDocs
       ? await Dartdoc.fromContext(config, packageBuilder)
       : await Dartdoc.withEmptyGenerator(config, packageBuilder);
   final results = await dartdoc.generateDocs();
 
-  final pubDataGenerator = PubDataGenerator(config.inputDir);
-  await pubDataGenerator.generate(results.packageGraph, config.output);
+  final pubDataGenerator =
+      PubDataGenerator(config.inputDir, pubResourceProvider);
+  pubDataGenerator.generate(results.packageGraph, config.output);
+
+  pubResourceProvider.writeFilesToDiskSync();
 }

pkg/pub_dartdoc/lib/pub_data_generator.dart

@@ -2,10 +2,9 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
-import 'dart:async';
 import 'dart:convert' as convert;
-import 'dart:io';
 
+import 'package:analyzer/file_system/file_system.dart';
 import 'package:dartdoc/dartdoc.dart';
 import 'package:path/path.dart' as p;
 
@@ -17,11 +16,11 @@ const fileName = 'pub-data.json';
 /// [PubDartdocData] instance.
 class PubDataGenerator {
   final String _inputDirectory;
+  final ResourceProvider _resourceProvider;
 
-  PubDataGenerator(this._inputDirectory);
+  PubDataGenerator(this._inputDirectory, this._resourceProvider);
 
-  Future<void> generate(
-      PackageGraph packageGraph, String outputDirectoryPath) async {
+  void generate(PackageGraph packageGraph, String outputDirectoryPath) {
     final modelElements = packageGraph.allCanonicalModelElements
         .where((elem) => elem.isPublic)
         .where((elem) => p.isWithin(_inputDirectory, elem.sourceFileName))
@@ -89,8 +88,9 @@ class PubDataGenerator {
         PubDartdocData(coverage: coverage, apiElements: apiElements);
 
     final fileName = 'pub-data.json';
-    final outputFile = File(p.join(outputDirectoryPath, fileName));
-    await outputFile.writeAsString(convert.json.encode(extract.toJson()));
+    _resourceProvider
+        .getFile(p.join(outputDirectoryPath, fileName))
+        .writeAsStringSync(convert.json.encode(extract.toJson()));
   }
 
   // Inherited member, should not show up in pub-data.json

pkg/pub_dartdoc/lib/src/pub_hooks.dart

@@ -0,0 +1,198 @@
+// Copyright (c) 2021, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:convert';
+import 'dart:io' as io;
+
+import 'package:analyzer/dart/element/element.dart';
+import 'package:analyzer/file_system/file_system.dart';
+import 'package:analyzer/file_system/memory_file_system.dart';
+// ignore: implementation_imports
+import 'package:analyzer/src/generated/sdk.dart';
+// ignore: implementation_imports
+import 'package:analyzer/src/generated/source.dart';
+import 'package:dartdoc/dartdoc.dart';
+import 'package:path/path.dart' as p;
+import 'package:watcher/watcher.dart';
+
+const _maxFileCount = 10000;
+const _maxTotalLengthBytes = 10 * 1024 * 1024;
+
+/// Creates an overlay file system with binary file support on top
+/// of the input sources.
+///
+/// TODO: Use a propr overlay in-memory filesystem with binary support,
+///       instead of overriding file writes in the output path.
+class PubResourceProvider implements ResourceProvider {
+  final ResourceProvider _defaultProvider;
+  final _memoryResourceProvider = MemoryResourceProvider();
+  String _outputPath;
+  int _fileCount = 0;
+  int _totalLengthBytes = 0;
+
+  PubResourceProvider(this._defaultProvider);
+
+  /// Writes in-memory files to disk.
+  void writeFilesToDiskSync() {
+    void storeFolder(Folder rs) {
+      for (final c in rs.getChildren()) {
+        if (c is Folder) {
+          storeFolder(c);
+        } else if (c is File) {
+          final file = io.File(c.path);
+          file.parent.createSync(recursive: true);
+          file.writeAsBytes(c.readAsBytesSync());
+        }
+      }
+    }
+
+    storeFolder(_memoryResourceProvider.getFolder(_outputPath));
+  }
+
+  /// Checks if we have reached any file write limit before storing the bytes.
+  void _aboutToWriteBytes(int length) {
+    _fileCount++;
+    _totalLengthBytes += length;
+    if (_fileCount > _maxFileCount) {
+      throw AssertionError(
+          'Reached $_maxFileCount files in the output directory.');
+    }
+    if (_totalLengthBytes > _maxTotalLengthBytes) {
+      throw AssertionError(
+          'Reached $_maxTotalLengthBytes bytes in the output directory.');
+    }
+  }
+
+  void setOutput(String value) {
+    _outputPath = value;
+  }
+
+  bool _isOutput(String path) {
+    return _outputPath != null &&
+        (path == _outputPath || p.isWithin(_outputPath, path));
+  }
+
+  ResourceProvider _rp(String path) =>
+      _isOutput(path) ? _memoryResourceProvider : _defaultProvider;
+
+  @override
+  File getFile(String path) => _File(this, _rp(path).getFile(path));
+
+  @override
+  Folder getFolder(String path) => _rp(path).getFolder(path);
+
+  @override
+  Future<List<int>> getModificationTimes(List<Source> sources) async {
+    return _defaultProvider.getModificationTimes(sources);
+  }
+
+  @override
+  Resource getResource(String path) => _rp(path).getResource(path);
+
+  @override
+  Folder getStateLocation(String pluginId) {
+    return _defaultProvider.getStateLocation(pluginId);
+  }
+
+  @override
+  p.Context get pathContext => _defaultProvider.pathContext;
+}
+
+class _File implements File {
+  final PubResourceProvider _provider;
+  final File _delegate;
+  _File(this._provider, this._delegate);
+
+  @override
+  Stream<WatchEvent> get changes => _delegate.changes;
+
+  @override
+  File copyTo(Folder parentFolder) => _delegate.copyTo(parentFolder);
+
+  @override
+  Source createSource([Uri uri]) => _delegate.createSource(uri);
+
+  @override
+  void delete() => _delegate.delete();
+
+  @override
+  bool get exists => _delegate.exists;
+
+  @override
+  bool isOrContains(String path) => _delegate.isOrContains(path);
+
+  @override
+  int get lengthSync => _delegate.lengthSync;
+
+  @override
+  int get modificationStamp => _delegate.modificationStamp;
+
+  @override
+  Folder get parent => _delegate.parent2;
+
+  @override
+  Folder get parent2 => _delegate.parent2;
+
+  @override
+  String get path => _delegate.path;
+
+  @override
+  ResourceProvider get provider => _delegate.provider;
+
+  @override
+  List<int> readAsBytesSync() => _delegate.readAsBytesSync();
+
+  @override
+  String readAsStringSync() => _delegate.readAsStringSync();
+
+  @override
+  File renameSync(String newPath) => _delegate.renameSync(newPath);
+
+  @override
+  Resource resolveSymbolicLinksSync() => _delegate.resolveSymbolicLinksSync();
+
+  @override
+  String get shortName => _delegate.shortName;
+
+  @override
+  Uri toUri() => _delegate.toUri();
+
+  @override
+  void writeAsBytesSync(List<int> bytes) {
+    _provider._aboutToWriteBytes(bytes.length);
+    _delegate.writeAsBytesSync(bytes);
+  }
+
+  @override
+  void writeAsStringSync(String content) {
+    writeAsBytesSync(utf8.encode(content));
+  }
+}
+
+/// Allows the override of [resourceProvider].
+class PubPackageMetaProvider implements PackageMetaProvider {
+  final PackageMetaProvider _delegate;
+  final ResourceProvider _resourceProvider;
+
+  PubPackageMetaProvider(this._delegate, this._resourceProvider);
+
+  @override
+  DartSdk get defaultSdk => _delegate.defaultSdk;
+
+  @override
+  Folder get defaultSdkDir => _delegate.defaultSdkDir;
+
+  @override
+  PackageMeta fromDir(Folder dir) => _delegate.fromDir(dir);
+
+  @override
+  PackageMeta fromElement(LibraryElement library, String s) =>
+      _delegate.fromElement(library, s);
+
+  @override
+  PackageMeta fromFilename(String s) => _delegate.fromFilename(s);
+
+  @override
+  ResourceProvider get resourceProvider => _resourceProvider;
+}

pkg/pub_dartdoc/pubspec.lock

@@ -9,7 +9,7 @@ packages:
     source: hosted
     version: "19.0.0"
   analyzer:
-    dependency: transitive
+    dependency: "direct main"
     description:
       name: analyzer
       url: "https://pub.dartlang.org"

pkg/pub_dartdoc/pubspec.yaml

@@ -4,6 +4,7 @@ description: The customized dartdoc for pub.dartlang.org.
 environment:
   sdk: ">=2.6.0 <3.0.0"
 dependencies:
+  analyzer: any # whatever dartdoc supports
   path: ^1.8.0
   pub_dartdoc_data:
     path: ../pub_dartdoc_data

pkg/pub_dartdoc/test/file_limit_test.dart

@@ -0,0 +1,30 @@
+// Copyright (c) 2021, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'package:analyzer/file_system/memory_file_system.dart';
+import 'package:test/test.dart';
+
+import 'package:pub_dartdoc/src/pub_hooks.dart';
+
+void main() {
+  test('limit number of files', () {
+    final provider = PubResourceProvider(MemoryResourceProvider());
+
+    for (var i = 0; i < 10000; i++) {
+      provider.getFile('/tmp/$i.txt').writeAsStringSync('x');
+    }
+
+    expect(() => provider.getFile('/tmp/next.txt').writeAsStringSync('next'),
+        throwsA(isA<AssertionError>()));
+  });
+
+  test('limit total bytes', () {
+    final provider = PubResourceProvider(MemoryResourceProvider());
+    provider
+        .getFile('/tmp/1')
+        .writeAsBytesSync(List<int>.filled(10 * 1024 * 1024, 0));
+    expect(() => provider.getFile('/tmp/2').writeAsBytesSync(<int>[0]),
+        throwsA(isA<AssertionError>()));
+  });
+}