Skip to content

Commit 4d55fb7

Browse files
isoosisoos
committed
Limit file count and total length in pkg/pub_dartdoc.
1 parent f0d9e61 commit 4d55fb7

File tree

6 files changed

+251
-12
lines changed

6 files changed

+251
-12
lines changed

‎pkg/pub_dartdoc/bin/pub_dartdoc.dart

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,20 +6,33 @@ import 'package:dartdoc/dartdoc.dart';
66
import 'package:dartdoc/options.dart';
77

88
import 'package:pub_dartdoc/pub_data_generator.dart';
9+
import 'package:pub_dartdoc/src/pub_hooks.dart';
910

1011
Future<void> main(List<String> arguments) async {
11-
final config = await parseOptions(pubPackageMetaProvider, arguments);
12+
// pub hooks
13+
final pubResourceProvider =
14+
PubResourceProvider(pubPackageMetaProvider.resourceProvider);
15+
final pubMetaProvider =
16+
PubPackageMetaProvider(pubPackageMetaProvider, pubResourceProvider);
17+
18+
// dartdoc initialization
19+
final config = await parseOptions(pubMetaProvider, arguments);
1220
if (config == null) {
1321
throw ArgumentError();
1422
}
23+
pubResourceProvider.setOutput(config.output);
24+
1525
final packageConfigProvider = PhysicalPackageConfigProvider();
1626
final packageBuilder =
17-
PubPackageBuilder(config, pubPackageMetaProvider, packageConfigProvider);
27+
PubPackageBuilder(config, pubMetaProvider, packageConfigProvider);
1828
final dartdoc = config.generateDocs
1929
? await Dartdoc.fromContext(config, packageBuilder)
2030
: await Dartdoc.withEmptyGenerator(config, packageBuilder);
2131
final results = await dartdoc.generateDocs();
2232

23-
final pubDataGenerator = PubDataGenerator(config.inputDir);
24-
await pubDataGenerator.generate(results.packageGraph, config.output);
33+
final pubDataGenerator =
34+
PubDataGenerator(config.inputDir, pubResourceProvider);
35+
pubDataGenerator.generate(results.packageGraph, config.output);
36+
37+
pubResourceProvider.writeFilesToDiskSync();
2538
}

‎pkg/pub_dartdoc/lib/pub_data_generator.dart

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,9 @@
22
// for details. All rights reserved. Use of this source code is governed by a
33
// BSD-style license that can be found in the LICENSE file.
44

5-
import 'dart:async';
65
import 'dart:convert' as convert;
7-
import 'dart:io';
86

7+
import 'package:analyzer/file_system/file_system.dart';
98
import 'package:dartdoc/dartdoc.dart';
109
import 'package:path/path.dart' as p;
1110

@@ -17,11 +16,11 @@ const fileName = 'pub-data.json';
1716
/// [PubDartdocData] instance.
1817
class PubDataGenerator {
1918
final String _inputDirectory;
19+
final ResourceProvider _resourceProvider;
2020

21-
PubDataGenerator(this._inputDirectory);
21+
PubDataGenerator(this._inputDirectory, this._resourceProvider);
2222

23-
Future<void> generate(
24-
PackageGraph packageGraph, String outputDirectoryPath) async {
23+
void generate(PackageGraph packageGraph, String outputDirectoryPath) {
2524
final modelElements = packageGraph.allCanonicalModelElements
2625
.where((elem) => elem.isPublic)
2726
.where((elem) => p.isWithin(_inputDirectory, elem.sourceFileName))
@@ -89,8 +88,9 @@ class PubDataGenerator {
8988
PubDartdocData(coverage: coverage, apiElements: apiElements);
9089

9190
final fileName = 'pub-data.json';
92-
final outputFile = File(p.join(outputDirectoryPath, fileName));
93-
await outputFile.writeAsString(convert.json.encode(extract.toJson()));
91+
_resourceProvider
92+
.getFile(p.join(outputDirectoryPath, fileName))
93+
.writeAsStringSync(convert.json.encode(extract.toJson()));
9494
}
9595

9696
// Inherited member, should not show up in pub-data.json

‎pkg/pub_dartdoc/lib/src/pub_hooks.dart

Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
// Copyright (c) 2021, the Dart project authors. Please see the AUTHORS file
2+
// for details. All rights reserved. Use of this source code is governed by a
3+
// BSD-style license that can be found in the LICENSE file.
4+
5+
import 'dart:convert';
6+
import 'dart:io' as io;
7+
8+
import 'package:analyzer/dart/element/element.dart';
9+
import 'package:analyzer/file_system/file_system.dart';
10+
import 'package:analyzer/file_system/memory_file_system.dart';
11+
// ignore: implementation_imports
12+
import 'package:analyzer/src/generated/sdk.dart';
13+
// ignore: implementation_imports
14+
import 'package:analyzer/src/generated/source.dart';
15+
import 'package:dartdoc/dartdoc.dart';
16+
import 'package:path/path.dart' as p;
17+
import 'package:watcher/watcher.dart';
18+
19+
const _maxFileCount = 10000;
20+
const _maxTotalLengthBytes = 10 * 1024 * 1024;
21+
22+
/// Creates an overlay file system with binary file support on top
23+
/// of the input sources.
24+
class PubResourceProvider implements ResourceProvider {
25+
final ResourceProvider _defaultProvider;
26+
final _memoryResourceProvider = MemoryResourceProvider();
27+
String _outputPath;
28+
int _fileCount = 0;
29+
int _totalLengthBytes = 0;
30+
31+
PubResourceProvider(this._defaultProvider);
32+
33+
/// Writes in-memory files to disk.
34+
void writeFilesToDiskSync() {
35+
void storeFolder(Folder rs) {
36+
for (final c in rs.getChildren()) {
37+
if (c is Folder) {
38+
storeFolder(c);
39+
} else if (c is File) {
40+
final file = io.File(c.path);
41+
file.parent.createSync(recursive: true);
42+
file.writeAsBytes(c.readAsBytesSync());
43+
}
44+
}
45+
}
46+
47+
storeFolder(_memoryResourceProvider.getFolder(_outputPath));
48+
}
49+
50+
/// Checks if we have reached any file write limit before storing the bytes.
51+
void _aboutToWriteBytes(int length) {
52+
_fileCount++;
53+
_totalLengthBytes += length;
54+
if (_fileCount > _maxFileCount) {
55+
throw AssertionError(
56+
'Reached $_maxFileCount files in the output directory.');
57+
}
58+
if (_totalLengthBytes > _maxTotalLengthBytes) {
59+
throw AssertionError(
60+
'Reached $_maxTotalLengthBytes bytes in the output directory.');
61+
}
62+
}
63+
64+
void setOutput(String value) {
65+
_outputPath = value;
66+
}
67+
68+
bool _isOutput(String path) {
69+
return _outputPath != null &&
70+
(path == _outputPath || p.isWithin(_outputPath, path));
71+
}
72+
73+
ResourceProvider _rp(String path) =>
74+
_isOutput(path) ? _memoryResourceProvider : _defaultProvider;
75+
76+
@override
77+
File getFile(String path) => _File(this, _rp(path).getFile(path));
78+
79+
@override
80+
Folder getFolder(String path) => _rp(path).getFolder(path);
81+
82+
@override
83+
Future<List<int>> getModificationTimes(List<Source> sources) async {
84+
return _defaultProvider.getModificationTimes(sources);
85+
}
86+
87+
@override
88+
Resource getResource(String path) => _rp(path).getResource(path);
89+
90+
@override
91+
Folder getStateLocation(String pluginId) {
92+
return _defaultProvider.getStateLocation(pluginId);
93+
}
94+
95+
@override
96+
p.Context get pathContext => _defaultProvider.pathContext;
97+
}
98+
99+
class _File implements File {
100+
final PubResourceProvider _provider;
101+
final File _delegate;
102+
_File(this._provider, this._delegate);
103+
104+
@override
105+
Stream<WatchEvent> get changes => _delegate.changes;
106+
107+
@override
108+
File copyTo(Folder parentFolder) => _delegate.copyTo(parentFolder);
109+
110+
@override
111+
Source createSource([Uri uri]) => _delegate.createSource(uri);
112+
113+
@override
114+
void delete() => _delegate.delete();
115+
116+
@override
117+
bool get exists => _delegate.exists;
118+
119+
@override
120+
bool isOrContains(String path) => _delegate.isOrContains(path);
121+
122+
@override
123+
int get lengthSync => _delegate.lengthSync;
124+
125+
@override
126+
int get modificationStamp => _delegate.modificationStamp;
127+
128+
@override
129+
Folder get parent => _delegate.parent2;
130+
131+
@override
132+
Folder get parent2 => _delegate.parent2;
133+
134+
@override
135+
String get path => _delegate.path;
136+
137+
@override
138+
ResourceProvider get provider => _delegate.provider;
139+
140+
@override
141+
List<int> readAsBytesSync() => _delegate.readAsBytesSync();
142+
143+
@override
144+
String readAsStringSync() => _delegate.readAsStringSync();
145+
146+
@override
147+
File renameSync(String newPath) => _delegate.renameSync(newPath);
148+
149+
@override
150+
Resource resolveSymbolicLinksSync() => _delegate.resolveSymbolicLinksSync();
151+
152+
@override
153+
String get shortName => _delegate.shortName;
154+
155+
@override
156+
Uri toUri() => _delegate.toUri();
157+
158+
@override
159+
void writeAsBytesSync(List<int> bytes) {
160+
_provider._aboutToWriteBytes(bytes.length);
161+
_delegate.writeAsBytesSync(bytes);
162+
}
163+
164+
@override
165+
void writeAsStringSync(String content) {
166+
writeAsBytesSync(utf8.encode(content));
167+
}
168+
}
169+
170+
/// Allows the override of [resourceProvider].
171+
class PubPackageMetaProvider implements PackageMetaProvider {
172+
final PackageMetaProvider _delegate;
173+
final ResourceProvider _resourceProvider;
174+
175+
PubPackageMetaProvider(this._delegate, this._resourceProvider);
176+
177+
@override
178+
DartSdk get defaultSdk => _delegate.defaultSdk;
179+
180+
@override
181+
Folder get defaultSdkDir => _delegate.defaultSdkDir;
182+
183+
@override
184+
PackageMeta fromDir(Folder dir) => _delegate.fromDir(dir);
185+
186+
@override
187+
PackageMeta fromElement(LibraryElement library, String s) =>
188+
_delegate.fromElement(library, s);
189+
190+
@override
191+
PackageMeta fromFilename(String s) => _delegate.fromFilename(s);
192+
193+
@override
194+
ResourceProvider get resourceProvider => _resourceProvider;
195+
}

‎pkg/pub_dartdoc/pubspec.lock

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ packages:
99
source: hosted
1010
version: "19.0.0"
1111
analyzer:
12-
dependency: transitive
12+
dependency: "direct main"
1313
description:
1414
name: analyzer
1515
url: "https://pub.dartlang.org"

‎pkg/pub_dartdoc/pubspec.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ description: The customized dartdoc for pub.dartlang.org.
44
environment:
55
sdk: ">=2.6.0 <3.0.0"
66
dependencies:
7+
analyzer: any # whatever dartdoc supports
78
path: ^1.8.0
89
pub_dartdoc_data:
910
path: ../pub_dartdoc_data

‎pkg/pub_dartdoc/test/file_limit_test.dart

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
// Copyright (c) 2021, the Dart project authors. Please see the AUTHORS file
2+
// for details. All rights reserved. Use of this source code is governed by a
3+
// BSD-style license that can be found in the LICENSE file.
4+
5+
import 'package:analyzer/file_system/memory_file_system.dart';
6+
import 'package:test/test.dart';
7+
8+
import 'package:pub_dartdoc/src/pub_hooks.dart';
9+
10+
void main() {
11+
test('limit number of files', () {
12+
final provider = PubResourceProvider(MemoryResourceProvider());
13+
14+
for (var i = 0; i < 10000; i++) {
15+
provider.getFile('/tmp/$i.txt').writeAsStringSync('x');
16+
}
17+
18+
expect(() => provider.getFile('/tmp/next.txt').writeAsStringSync('next'),
19+
throwsA(isA<AssertionError>()));
20+
});
21+
22+
test('limit total bytes', () {
23+
final provider = PubResourceProvider(MemoryResourceProvider());
24+
provider
25+
.getFile('/tmp/1')
26+
.writeAsBytesSync(List<int>.filled(10 * 1024 * 1024, 0));
27+
expect(() => provider.getFile('/tmp/2').writeAsBytesSync(<int>[0]),
28+
throwsA(isA<AssertionError>()));
29+
});
30+
}

0 commit comments

Comments
 (0)