From 9b2bfa7646c153c19f9b301f80e7410f31b6e01d Mon Sep 17 00:00:00 2001 From: Janice Collins Date: Mon, 11 Dec 2017 14:59:37 -0800 Subject: [PATCH 01/11] First revision --- pubspec.lock | 14 ++- pubspec.yaml | 1 + tool/grind.dart | 277 +++++++++++++++++++++++++++--------------------- 3 files changed, 170 insertions(+), 122 deletions(-) diff --git a/pubspec.lock b/pubspec.lock index ecf72fc41e..566f015931 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -73,6 +73,12 @@ packages: url: "https://pub.dartlang.org" source: hosted version: "0.14.1" + dhttpd: + description: + name: dhttpd + url: "https://pub.dartlang.org" + source: hosted + version: "0.3.1" front_end: description: name: front_end @@ -246,7 +252,13 @@ packages: name: shelf url: "https://pub.dartlang.org" source: hosted - version: "0.7.1" + version: "0.6.8" + shelf_cors: + description: + name: shelf_cors + url: "https://pub.dartlang.org" + source: hosted + version: "0.2.1" shelf_packages_handler: description: name: shelf_packages_handler diff --git a/pubspec.yaml b/pubspec.yaml index df2514153d..b019fb4d9f 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -26,6 +26,7 @@ dependencies: tuple: ^1.0.1 yaml: ^2.1.0 dev_dependencies: + dhttpd: ^0.3.1 grinder: ^0.8.0 http: ^0.11.0 meta: ^1.0.0 diff --git a/tool/grind.dart b/tool/grind.dart index 755f341b86..655311ec9f 100644 --- a/tool/grind.dart +++ b/tool/grind.dart @@ -3,20 +3,92 @@ // BSD-style license that can be found in the LICENSE file. import 'dart:async'; +import 'dart:convert'; import 'dart:io' hide ProcessException; -import 'package:dartdoc/dartdoc.dart' show defaultOutDir; import 'package:dartdoc/src/io_utils.dart'; import 'package:grinder/grinder.dart'; -import 'package:html/dom.dart'; -import 'package:html/parser.dart' show parse; import 'package:path/path.dart' as path; import 'package:yaml/yaml.dart' as yaml; main([List args]) => grind(args); -final Directory docsDir = - new Directory(path.join('${Directory.systemTemp.path}', defaultOutDir)); +final Directory dartdocDocsDir = Directory.systemTemp.createTempSync('dartdoc'); +final Directory sdkDocsDir = Directory.systemTemp.createTempSync('sdkdocs'); +final Directory flutterDir = Directory.systemTemp.createTempSync('flutter'); +final Directory flutterDirDevTools = new Directory(path.join(flutterDir.path, 'dev', 'tools')); +final String pubPath = path.join(path.dirname(Platform.resolvedExecutable), + 'pub'); + +final RegExp quotables = new RegExp(r'[ "\r\n\$]'); + +// from flutter:dev/tools/dartdoc.dart, modified +void printStream(Stream> stream, Stdout output, { String prefix: '' }) { + assert(prefix != null); + stream + .transform(UTF8.decoder) + .transform(const LineSplitter()) + .listen((String line) { + output.write('$prefix$line'.trim()); + output.write('\n'); + }); +} + +class SubprocessLauncher { + final String context; + Map _environment; + + Map get environment => _environment; + + SubprocessLauncher(this.context, [Map environment]) { + if (environment == null) this._environment = new Map(); + } + + /// A wrapper around start/await process.exitCode that will display the + /// output of the executable continuously and fail on non-zero exit codes. + /// Makes running programs in grinder similar to set -ex for bash, even on + /// Windows (though some of the bashisms will no longer make sense). + /// TODO(jcollins-g): move this to grinder? + Future runStreamed(String executable, List arguments, + {String workingDirectory}) async { + stderr.write('+ '); + if (workingDirectory != null) stderr.write('cd "$workingDirectory" && '); + if (environment != null) { + stderr.write(environment.keys.map((String key) { + if (environment[key].contains(quotables)) { + return "$key='${environment[key]}'"; + } else { + return "$key=${environment[key]}"; + } + }).join(' ')); + stderr.write(' '); + } + stderr.write('$executable'); + if (arguments.isNotEmpty) { + for (String arg in arguments) { + if (arg.contains(quotables)) { + stderr.write(" '$arg'"); + } else { + stderr.write(" $arg"); + } + } + } + if (workingDirectory != null) stderr.write(')'); + stderr.write('\n'); + Process process = await Process.start(executable, arguments, + workingDirectory: workingDirectory, + environment: environment); + + printStream(process.stdout, stdout, prefix: '$context: '); + printStream(process.stderr, stderr, prefix: '$context: '); + await process.exitCode; + + int exitCode = await process.exitCode; + if (exitCode != 0) { + fail("exitCode: $exitCode"); + } + } +} @Task('Analyze dartdoc to ensure there are no errors and warnings') analyze() { @@ -29,23 +101,85 @@ buildbot() => null; @Task('Generate docs for the Dart SDK') Future buildSdkDocs() async { - delete(docsDir); log('building SDK docs'); - Process process = await Process.start(Platform.resolvedExecutable, [ + var launcher = new SubprocessLauncher('dartdoc[sdk]'); + await launcher.runStreamed(Platform.resolvedExecutable, [ '--checked', 'bin/dartdoc.dart', '--output', - '${docsDir.path}', + '${sdkDocsDir.path}', '--sdk-docs', '--show-progress' ]); - stdout.addStream(process.stdout); - stderr.addStream(process.stderr); +} - int exitCode = await process.exitCode; - if (exitCode != 0) { - fail("exitCode: $exitCode"); - } +@Task('Serve generated SDK docs locally with dhttpd on port 8000') +@Depends(buildSdkDocs) +Future serveSdkDocs() async { + log('launching dhttpd on port 8000 for SDK'); + var launcher = new SubprocessLauncher('dhttpd[sdk]'); + await launcher.runStreamed(pubPath, [ + 'run', 'dhttpd', + '--port', + '8000', + '--path', + '${sdkDocsDir.path}', + ]); +} + +@Task('Serve generated Flutter docs locally with dhttpd on port 8001') +@Depends(buildFlutterDocs) +Future serveFlutterDocs() async { + log('launching dhttpd on port 8001 for Flutter'); + var launcher = new SubprocessLauncher('dhttpd[flutter]'); + await launcher.runStreamed(pubPath, ['get']); + await launcher.runStreamed(pubPath, [ + 'run', 'dhttpd', + '--port', + '8001', + '--path', + path.join(flutterDir.path, 'dev', 'docs', 'doc'), + ]); +} + +/// Creates a throwaway pub cache and returns the environment variables +/// necessary to use it. +Map _createThrowawayPubCache() { + final Directory pubCache = Directory.systemTemp.createTempSync('pubcache'); + final Directory pubCacheBin = new Directory(path.join(pubCache.path, 'bin')); + pubCacheBin.createSync(); + return new Map.fromIterables(['PUB_CACHE', 'PATH'], [pubCache.path, + [pubCacheBin.path, Platform.environment['PATH']].join(':')]); +} + +@Task('Build flutter docs') +Future buildFlutterDocs() async { + log('building flutter docs into: $flutterDir'); + var launcher = new SubprocessLauncher('dartdoc[flutter]', _createThrowawayPubCache()); + await launcher.runStreamed('git', [ + 'clone', + '--depth', '1', + 'https://github.com/flutter/flutter.git', + '.' + ], workingDirectory: flutterDir.path); + String flutterBin = path.join('bin', 'flutter'); + String flutterCacheDart = path.join(flutterDir.path, 'bin', 'cache', 'dart-sdk', 'bin', 'dart'); + String flutterCachePub = path.join(flutterDir.path, 'bin', 'cache', 'dart-sdk', 'bin', 'pub'); + await launcher.runStreamed(flutterBin, ['--version'], + workingDirectory: flutterDir.path, + ); + await launcher.runStreamed(flutterBin, ['precache'], + workingDirectory: flutterDir.path, + ); + await launcher.runStreamed(flutterCachePub, ['get'], + workingDirectory: path.join(flutterDir.path, 'dev', 'tools'), + ); + await launcher.runStreamed(flutterCachePub, ['global', 'activate', '-spath', '.']); + await launcher.runStreamed(flutterCacheDart, [path.join('dev', 'tools', 'dartdoc.dart')], + workingDirectory: flutterDir.path, + ); + String index = new File(path.join(flutterDir.path, 'dev', 'docs', 'doc', 'index.html')).readAsStringSync(); + stdout.write(index); } @Task('Checks that CHANGELOG mentions current version') @@ -75,32 +209,6 @@ _getPackageVersion() { return version; } -@Task('Check links') -checkLinks() { - bool foundError = false; - Set visited = new Set(); - final origin = 'testing/test_package_docs/'; - var start = 'index.html'; - - _doCheck(origin, visited, start, foundError); - _doFileCheck(origin, visited, foundError); - - if (foundError) exit(1); -} - -@Task('Check sdk links') -checkSdkLinks() { - bool foundError = false; - Set visited = new Set(); - final origin = '${docsDir.path}/'; - var start = 'index.html'; - - _doCheck(origin, visited, start, foundError); - _doFileCheck(origin, visited, foundError); - - if (foundError) exit(1); -} - @Task('Checks that version is matched in relevant places') checkVersionMatches() async { var version = _getPackageVersion(); @@ -187,17 +295,11 @@ test() { @Task('Generate docs for dartdoc') testDartdoc() { - delete(docsDir); - try { - log('running dartdoc'); - Dart.run('bin/dartdoc.dart', - arguments: ['--output', '${docsDir.path}'], vmArgs: ['--checked']); - - File indexHtml = joinFile(docsDir, ['index.html']); - if (!indexHtml.existsSync()) fail('docs not generated'); - } catch (e) { - rethrow; - } + var launcher = new SubprocessLauncher('dartdoc[test]'); + launcher.runStreamed(Platform.resolvedExecutable, + ['-c', 'bin/dartdoc.dart', '--output', dartdocDocsDir.path]); + File indexHtml = joinFile(sdkDocsDir, ['index.html']); + if (!indexHtml.existsSync()) fail('docs not generated'); } @Task('update test_package_docs') @@ -228,7 +330,7 @@ updateTestPackageDocs() { validateSdkDocs() { const expectedLibCount = 18; - File indexHtml = joinFile(docsDir, ['index.html']); + File indexHtml = joinFile(sdkDocsDir, ['index.html']); if (!indexHtml.existsSync()) { fail('no index.html found for SDK docs'); } @@ -243,7 +345,7 @@ validateSdkDocs() { // check for the existence of certain files/dirs var libsLength = - docsDir.listSync().where((fs) => fs.path.contains('dart-')).length; + sdkDocsDir.listSync().where((fs) => fs.path.contains('dart-')).length; if (libsLength != expectedLibCount) { fail('docs not generated for all the SDK libraries, ' 'expected $expectedLibCount directories, generated $libsLength directories'); @@ -251,7 +353,7 @@ validateSdkDocs() { log('$libsLength dart: libraries found'); var futureConstFile = - joinFile(docsDir, [path.join('dart-async', 'Future', 'Future.html')]); + joinFile(sdkDocsDir, [path.join('dart-async', 'Future', 'Future.html')]); if (!futureConstFile.existsSync()) { fail('no Future.html found for dart:async Future constructor'); } @@ -267,70 +369,3 @@ int _findCount(String str, String match) { } return count; } - -Stream dirContents(String dir) { - return new Directory(dir).list(recursive: true); -} - -void _doFileCheck(String origin, Set visited, bool error) { - String normalOrigin = path.normalize(origin); - dirContents(normalOrigin).toList().then((allFiles) { - bool foundIndex = false; - for (FileSystemEntity f in allFiles) { - if (f is Directory) continue; - var fullPath = path.normalize(f.path); - if (fullPath.startsWith("${normalOrigin}/static-assets/")) continue; - if (fullPath == "${normalOrigin}/index.json") { - foundIndex = true; - continue; - } - if (visited.contains(fullPath)) continue; - log(' * Orphaned: $fullPath'); - error = true; - } - if (!foundIndex) { - log(' * Not found: ${normalOrigin}/index.json'); - error = true; - } - }); -} - -void _doCheck( - String origin, Set visited, String pathToCheck, bool error, - [String source]) { - var fullPath = path.normalize("$origin$pathToCheck"); - if (visited.contains(fullPath)) return; - visited.add(fullPath); - - File file = new File("$fullPath"); - if (!file.existsSync()) { - // There is a deliberately broken link in one place. - if (!fullPath.endsWith("ftp:/ftp.myfakepackage.com/donthidemyschema")) { - error = true; - log(' * Not found: $fullPath from $source'); - } - return; - } - Document doc = parse(file.readAsStringSync()); - Element base = doc.querySelector('base'); - String baseHref; - if (base != null) { - baseHref = base.attributes['href']; - } - List links = doc.querySelectorAll('a'); - links - .map((link) => link.attributes['href']) - .where((href) => href != null) - .forEach((href) { - if (!href.startsWith('http') && !href.contains('#')) { - var full; - if (baseHref != null) { - full = '${path.dirname(pathToCheck)}/$baseHref/$href'; - } else { - full = '${path.dirname(pathToCheck)}/$href'; - } - var normalized = path.normalize(full); - _doCheck(origin, visited, normalized, error, pathToCheck); - } - }); -} From 45ffe336b4061e0abc7152f27e6edbc29a788611 Mon Sep 17 00:00:00 2001 From: Janice Collins Date: Mon, 11 Dec 2017 15:14:48 -0800 Subject: [PATCH 02/11] Move to a class based model --- tool/grind.dart | 50 ++++++++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/tool/grind.dart b/tool/grind.dart index 655311ec9f..dd3fd636a2 100644 --- a/tool/grind.dart +++ b/tool/grind.dart @@ -40,6 +40,8 @@ class SubprocessLauncher { Map get environment => _environment; + String get prefix => context.isNotEmpty ? '$context: ' : ''; + SubprocessLauncher(this.context, [Map environment]) { if (environment == null) this._environment = new Map(); } @@ -51,7 +53,7 @@ class SubprocessLauncher { /// TODO(jcollins-g): move this to grinder? Future runStreamed(String executable, List arguments, {String workingDirectory}) async { - stderr.write('+ '); + stderr.write('$prefix+ '); if (workingDirectory != null) stderr.write('cd "$workingDirectory" && '); if (environment != null) { stderr.write(environment.keys.map((String key) { @@ -79,8 +81,8 @@ class SubprocessLauncher { workingDirectory: workingDirectory, environment: environment); - printStream(process.stdout, stdout, prefix: '$context: '); - printStream(process.stderr, stderr, prefix: '$context: '); + printStream(process.stdout, stdout, prefix: prefix); + printStream(process.stderr, stderr, prefix: prefix); await process.exitCode; int exitCode = await process.exitCode; @@ -294,35 +296,33 @@ test() { } @Task('Generate docs for dartdoc') -testDartdoc() { +testDartdoc() async { var launcher = new SubprocessLauncher('dartdoc[test]'); - launcher.runStreamed(Platform.resolvedExecutable, - ['-c', 'bin/dartdoc.dart', '--output', dartdocDocsDir.path]); - File indexHtml = joinFile(sdkDocsDir, ['index.html']); + await launcher.runStreamed(Platform.resolvedExecutable, + ['--checked', 'bin/dartdoc.dart', '--output', dartdocDocsDir.path]); + File indexHtml = joinFile(dartdocDocsDir, ['index.html']); if (!indexHtml.existsSync()) fail('docs not generated'); } @Task('update test_package_docs') -updateTestPackageDocs() { - var options = new RunOptions(workingDirectory: 'testing/test_package'); +updateTestPackageDocs() async { + var launcher = new SubprocessLauncher('dartdoc[update-test-package]'); // This must be synced with ../test/compare_output_test.dart's // "Validate html output of test_package" test. - delete(getDir('testing/test_package_docs')); - Dart.run('../../bin/dartdoc.dart', - arguments: [ - '--auto-include-dependencies', - '--example-path-prefix', - 'examples', - '--no-include-source', - '--pretty-index-json', - '--hide-sdk-text', - '--exclude', - 'dart.async,dart.collection,dart.convert,dart.core,dart.math,dart.typed_data,package:meta/meta.dart', - '--output', - '../test_package_docs', - ], - runOptions: options, - vmArgs: ['--checked']); + await launcher.runStreamed(Platform.resolvedExecutable, [ + '--checked', + path.join('..', '..', 'bin', 'dartdoc.dart'), + '--auto-include-dependencies', + '--example-path-prefix', + 'examples', + '--no-include-source', + '--pretty-index-json', + '--hide-sdk-text', + '--exclude', + 'dart.async,dart.collection,dart.convert,dart.core,dart.math,dart.typed_data,package:meta/meta.dart', + '--output', + '../test_package_docs', + ], workingDirectory: path.join('testing', 'test_package_docs')); } @Task('Validate the SDK doc build.') From 6a2dd2916f5b5b919319cef196b7df82f51d65a5 Mon Sep 17 00:00:00 2001 From: Janice Collins Date: Mon, 11 Dec 2017 15:19:59 -0800 Subject: [PATCH 03/11] Debugging on update-test-package-docs --- .../test_package_docs/static-assets/readme.md | 19 +++++++++++++++++++ .../static-assets/sdk_footer_text.html | 4 ++++ tool/grind.dart | 6 +++++- 3 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 testing/test_package_docs/static-assets/readme.md create mode 100644 testing/test_package_docs/static-assets/sdk_footer_text.html diff --git a/testing/test_package_docs/static-assets/readme.md b/testing/test_package_docs/static-assets/readme.md new file mode 100644 index 0000000000..287f566e77 --- /dev/null +++ b/testing/test_package_docs/static-assets/readme.md @@ -0,0 +1,19 @@ +# highlight.js + +Generated from https://highlightjs.org/download/ on 2017-08-30 + +Included languages: + +* bash +* css +* dart +* java +* javascript +* json +* markdown +* objectivec +* ruby - dragged in by `yaml` - 🙄 +* shell +* swift +* xml - includes html +* yaml diff --git a/testing/test_package_docs/static-assets/sdk_footer_text.html b/testing/test_package_docs/static-assets/sdk_footer_text.html new file mode 100644 index 0000000000..e613e68048 --- /dev/null +++ b/testing/test_package_docs/static-assets/sdk_footer_text.html @@ -0,0 +1,4 @@ +• + + cc license + diff --git a/tool/grind.dart b/tool/grind.dart index dd3fd636a2..a6570eefb6 100644 --- a/tool/grind.dart +++ b/tool/grind.dart @@ -307,6 +307,10 @@ testDartdoc() async { @Task('update test_package_docs') updateTestPackageDocs() async { var launcher = new SubprocessLauncher('dartdoc[update-test-package]'); + var testPackageDocs = new Directory(path.join('testing', 'test_package_docs')); + var testPackage = new Directory(path.join('testing', 'test_package')); + await launcher.runStreamed(pubPath, ['get'], workingDirectory: testPackage.path); + delete(testPackageDocs); // This must be synced with ../test/compare_output_test.dart's // "Validate html output of test_package" test. await launcher.runStreamed(Platform.resolvedExecutable, [ @@ -322,7 +326,7 @@ updateTestPackageDocs() async { 'dart.async,dart.collection,dart.convert,dart.core,dart.math,dart.typed_data,package:meta/meta.dart', '--output', '../test_package_docs', - ], workingDirectory: path.join('testing', 'test_package_docs')); + ], workingDirectory: testPackage.path); } @Task('Validate the SDK doc build.') From 23e67b73ce1a53c4d202bc3e9d386d8d4f9bc7c1 Mon Sep 17 00:00:00 2001 From: Janice Collins Date: Mon, 11 Dec 2017 15:21:20 -0800 Subject: [PATCH 04/11] Simplify travis --- tool/travis.sh | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/tool/travis.sh b/tool/travis.sh index 580ce2c50d..ec4731f3b9 100755 --- a/tool/travis.sh +++ b/tool/travis.sh @@ -22,25 +22,7 @@ if [ "$DARTDOC_BOT" = "sdk-docs" ]; then elif [ "$DARTDOC_BOT" = "flutter" ]; then echo "Running flutter dartdoc bot" - # Verify that the libraries are error free. - pub run grinder analyze - - # Set up dartdoc so the flutter doc script can locate it. - pub global activate -spath . - - # Clone flutter. - rm -rf doc/flutter - git clone --depth 1 https://github.com/flutter/flutter.git doc/flutter - - # Build the flutter docs. - cd doc/flutter - ./bin/flutter --version - ./bin/flutter precache - ( cd dev/tools; pub get ) - ./bin/cache/dart-sdk/bin/dart dev/tools/dartdoc.dart - - # The above script validates the generation; we echo the main file here. - cat dev/docs/doc/index.html + pub run grinder build-flutter-docs else echo "Running main dartdoc bot" @@ -50,9 +32,6 @@ else # Run dartdoc on test_package. (cd testing/test_package; dart -c ../../bin/dartdoc.dart) - # Checks the test_package results. - pub run grinder check-links - # And on test_package_small. (cd testing/test_package_small; dart -c ../../bin/dartdoc.dart) From 3f9780ef3e9d9c278af80ca6c3f622cbb2ff81c9 Mon Sep 17 00:00:00 2001 From: Janice Collins Date: Mon, 11 Dec 2017 15:28:57 -0800 Subject: [PATCH 05/11] cosmetics / dartfmt --- tool/grind.dart | 144 ++++++++++++++++++++++++++++-------------------- 1 file changed, 84 insertions(+), 60 deletions(-) diff --git a/tool/grind.dart b/tool/grind.dart index a6570eefb6..d0058c8618 100644 --- a/tool/grind.dart +++ b/tool/grind.dart @@ -16,14 +16,16 @@ main([List args]) => grind(args); final Directory dartdocDocsDir = Directory.systemTemp.createTempSync('dartdoc'); final Directory sdkDocsDir = Directory.systemTemp.createTempSync('sdkdocs'); final Directory flutterDir = Directory.systemTemp.createTempSync('flutter'); -final Directory flutterDirDevTools = new Directory(path.join(flutterDir.path, 'dev', 'tools')); -final String pubPath = path.join(path.dirname(Platform.resolvedExecutable), - 'pub'); +final Directory flutterDirDevTools = + new Directory(path.join(flutterDir.path, 'dev', 'tools')); +final String pubPath = + path.join(path.dirname(Platform.resolvedExecutable), 'pub'); final RegExp quotables = new RegExp(r'[ "\r\n\$]'); // from flutter:dev/tools/dartdoc.dart, modified -void printStream(Stream> stream, Stdout output, { String prefix: '' }) { +void _printStream(Stream> stream, Stdout output, + {String prefix: ''}) { assert(prefix != null); stream .transform(UTF8.decoder) @@ -34,7 +36,22 @@ void printStream(Stream> stream, Stdout output, { String prefix: '' }) }); } -class SubprocessLauncher { +/// Creates a throwaway pub cache and returns the environment variables +/// necessary to use it. +Map _createThrowawayPubCache() { + final Directory pubCache = Directory.systemTemp.createTempSync('pubcache'); + final Directory pubCacheBin = new Directory(path.join(pubCache.path, 'bin')); + pubCacheBin.createSync(); + return new Map.fromIterables([ + 'PUB_CACHE', + 'PATH' + ], [ + pubCache.path, + [pubCacheBin.path, Platform.environment['PATH']].join(':') + ]); +} + +class _SubprocessLauncher { final String context; Map _environment; @@ -42,7 +59,7 @@ class SubprocessLauncher { String get prefix => context.isNotEmpty ? '$context: ' : ''; - SubprocessLauncher(this.context, [Map environment]) { + _SubprocessLauncher(this.context, [Map environment]) { if (environment == null) this._environment = new Map(); } @@ -78,11 +95,10 @@ class SubprocessLauncher { if (workingDirectory != null) stderr.write(')'); stderr.write('\n'); Process process = await Process.start(executable, arguments, - workingDirectory: workingDirectory, - environment: environment); + workingDirectory: workingDirectory, environment: environment); - printStream(process.stdout, stdout, prefix: prefix); - printStream(process.stderr, stderr, prefix: prefix); + _printStream(process.stdout, stdout, prefix: prefix); + _printStream(process.stderr, stderr, prefix: prefix); await process.exitCode; int exitCode = await process.exitCode; @@ -104,7 +120,7 @@ buildbot() => null; @Task('Generate docs for the Dart SDK') Future buildSdkDocs() async { log('building SDK docs'); - var launcher = new SubprocessLauncher('dartdoc[sdk]'); + var launcher = new _SubprocessLauncher('build-sdk-docs'); await launcher.runStreamed(Platform.resolvedExecutable, [ '--checked', 'bin/dartdoc.dart', @@ -119,9 +135,10 @@ Future buildSdkDocs() async { @Depends(buildSdkDocs) Future serveSdkDocs() async { log('launching dhttpd on port 8000 for SDK'); - var launcher = new SubprocessLauncher('dhttpd[sdk]'); + var launcher = new _SubprocessLauncher('serve-sdk-docs'); await launcher.runStreamed(pubPath, [ - 'run', 'dhttpd', + 'run', + 'dhttpd', '--port', '8000', '--path', @@ -133,10 +150,11 @@ Future serveSdkDocs() async { @Depends(buildFlutterDocs) Future serveFlutterDocs() async { log('launching dhttpd on port 8001 for Flutter'); - var launcher = new SubprocessLauncher('dhttpd[flutter]'); + var launcher = new _SubprocessLauncher('serve-flutter-docs'); await launcher.runStreamed(pubPath, ['get']); await launcher.runStreamed(pubPath, [ - 'run', 'dhttpd', + 'run', + 'dhttpd', '--port', '8001', '--path', @@ -144,43 +162,44 @@ Future serveFlutterDocs() async { ]); } -/// Creates a throwaway pub cache and returns the environment variables -/// necessary to use it. -Map _createThrowawayPubCache() { - final Directory pubCache = Directory.systemTemp.createTempSync('pubcache'); - final Directory pubCacheBin = new Directory(path.join(pubCache.path, 'bin')); - pubCacheBin.createSync(); - return new Map.fromIterables(['PUB_CACHE', 'PATH'], [pubCache.path, - [pubCacheBin.path, Platform.environment['PATH']].join(':')]); -} - @Task('Build flutter docs') Future buildFlutterDocs() async { log('building flutter docs into: $flutterDir'); - var launcher = new SubprocessLauncher('dartdoc[flutter]', _createThrowawayPubCache()); - await launcher.runStreamed('git', [ - 'clone', - '--depth', '1', - 'https://github.com/flutter/flutter.git', - '.' - ], workingDirectory: flutterDir.path); + var launcher = + new _SubprocessLauncher('build-flutter-docs', _createThrowawayPubCache()); + await launcher.runStreamed('git', + ['clone', '--depth', '1', 'https://github.com/flutter/flutter.git', '.'], + workingDirectory: flutterDir.path); String flutterBin = path.join('bin', 'flutter'); - String flutterCacheDart = path.join(flutterDir.path, 'bin', 'cache', 'dart-sdk', 'bin', 'dart'); - String flutterCachePub = path.join(flutterDir.path, 'bin', 'cache', 'dart-sdk', 'bin', 'pub'); - await launcher.runStreamed(flutterBin, ['--version'], + String flutterCacheDart = + path.join(flutterDir.path, 'bin', 'cache', 'dart-sdk', 'bin', 'dart'); + String flutterCachePub = + path.join(flutterDir.path, 'bin', 'cache', 'dart-sdk', 'bin', 'pub'); + await launcher.runStreamed( + flutterBin, + ['--version'], workingDirectory: flutterDir.path, ); - await launcher.runStreamed(flutterBin, ['precache'], + await launcher.runStreamed( + flutterBin, + ['precache'], workingDirectory: flutterDir.path, ); - await launcher.runStreamed(flutterCachePub, ['get'], + await launcher.runStreamed( + flutterCachePub, + ['get'], workingDirectory: path.join(flutterDir.path, 'dev', 'tools'), ); - await launcher.runStreamed(flutterCachePub, ['global', 'activate', '-spath', '.']); - await launcher.runStreamed(flutterCacheDart, [path.join('dev', 'tools', 'dartdoc.dart')], + await launcher + .runStreamed(flutterCachePub, ['global', 'activate', '-spath', '.']); + await launcher.runStreamed( + flutterCacheDart, + [path.join('dev', 'tools', 'dartdoc.dart')], workingDirectory: flutterDir.path, ); - String index = new File(path.join(flutterDir.path, 'dev', 'docs', 'doc', 'index.html')).readAsStringSync(); + String index = + new File(path.join(flutterDir.path, 'dev', 'docs', 'doc', 'index.html')) + .readAsStringSync(); stdout.write(index); } @@ -289,15 +308,15 @@ publish() async { } @Task('Run all the tests.') -test() { +test() async { // `pub run test` is a bit slower than running an `test_all.dart` script // But it provides more useful output in the case of failures. - return Pub.runAsync('test'); + await new _SubprocessLauncher('test').runStreamed(pubPath, ['run', 'test']); } @Task('Generate docs for dartdoc') testDartdoc() async { - var launcher = new SubprocessLauncher('dartdoc[test]'); + var launcher = new _SubprocessLauncher('test-dartdoc'); await launcher.runStreamed(Platform.resolvedExecutable, ['--checked', 'bin/dartdoc.dart', '--output', dartdocDocsDir.path]); File indexHtml = joinFile(dartdocDocsDir, ['index.html']); @@ -306,27 +325,32 @@ testDartdoc() async { @Task('update test_package_docs') updateTestPackageDocs() async { - var launcher = new SubprocessLauncher('dartdoc[update-test-package]'); - var testPackageDocs = new Directory(path.join('testing', 'test_package_docs')); + var launcher = new _SubprocessLauncher('update-test-package-docs'); + var testPackageDocs = + new Directory(path.join('testing', 'test_package_docs')); var testPackage = new Directory(path.join('testing', 'test_package')); - await launcher.runStreamed(pubPath, ['get'], workingDirectory: testPackage.path); + await launcher.runStreamed(pubPath, ['get'], + workingDirectory: testPackage.path); delete(testPackageDocs); // This must be synced with ../test/compare_output_test.dart's // "Validate html output of test_package" test. - await launcher.runStreamed(Platform.resolvedExecutable, [ - '--checked', - path.join('..', '..', 'bin', 'dartdoc.dart'), - '--auto-include-dependencies', - '--example-path-prefix', - 'examples', - '--no-include-source', - '--pretty-index-json', - '--hide-sdk-text', - '--exclude', - 'dart.async,dart.collection,dart.convert,dart.core,dart.math,dart.typed_data,package:meta/meta.dart', - '--output', - '../test_package_docs', - ], workingDirectory: testPackage.path); + await launcher.runStreamed( + Platform.resolvedExecutable, + [ + '--checked', + path.join('..', '..', 'bin', 'dartdoc.dart'), + '--auto-include-dependencies', + '--example-path-prefix', + 'examples', + '--no-include-source', + '--pretty-index-json', + '--hide-sdk-text', + '--exclude', + 'dart.async,dart.collection,dart.convert,dart.core,dart.math,dart.typed_data,package:meta/meta.dart', + '--output', + '../test_package_docs', + ], + workingDirectory: testPackage.path); } @Task('Validate the SDK doc build.') From 0d17158b88b4d2dd1e1a4f33a568db33559e26f4 Mon Sep 17 00:00:00 2001 From: Janice Collins Date: Mon, 11 Dec 2017 15:32:08 -0800 Subject: [PATCH 06/11] Windows doesn't like using pub in a literal path --- tool/grind.dart | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tool/grind.dart b/tool/grind.dart index d0058c8618..bb59f5eac2 100644 --- a/tool/grind.dart +++ b/tool/grind.dart @@ -18,8 +18,7 @@ final Directory sdkDocsDir = Directory.systemTemp.createTempSync('sdkdocs'); final Directory flutterDir = Directory.systemTemp.createTempSync('flutter'); final Directory flutterDirDevTools = new Directory(path.join(flutterDir.path, 'dev', 'tools')); -final String pubPath = - path.join(path.dirname(Platform.resolvedExecutable), 'pub'); +final String pubPath = 'pub'; final RegExp quotables = new RegExp(r'[ "\r\n\$]'); From 63e4bfd38b12463299e4bea99423f00ca856759f Mon Sep 17 00:00:00 2001 From: Janice Collins Date: Mon, 11 Dec 2017 15:36:22 -0800 Subject: [PATCH 07/11] windows fixing --- tool/grind.dart | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tool/grind.dart b/tool/grind.dart index bb59f5eac2..3215115793 100644 --- a/tool/grind.dart +++ b/tool/grind.dart @@ -18,7 +18,8 @@ final Directory sdkDocsDir = Directory.systemTemp.createTempSync('sdkdocs'); final Directory flutterDir = Directory.systemTemp.createTempSync('flutter'); final Directory flutterDirDevTools = new Directory(path.join(flutterDir.path, 'dev', 'tools')); -final String pubPath = 'pub'; +final String pubPath = + path.join(path.dirname(Platform.resolvedExecutable), Platform.isWindows ? 'pub.bat' : 'pub'); final RegExp quotables = new RegExp(r'[ "\r\n\$]'); From fc15a7926713015dc231e9dd3283adcb8b88f066 Mon Sep 17 00:00:00 2001 From: Janice Collins Date: Mon, 11 Dec 2017 15:38:10 -0800 Subject: [PATCH 08/11] fix windows some more --- tool/grind.dart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tool/grind.dart b/tool/grind.dart index 3215115793..46dd807ed5 100644 --- a/tool/grind.dart +++ b/tool/grind.dart @@ -18,8 +18,8 @@ final Directory sdkDocsDir = Directory.systemTemp.createTempSync('sdkdocs'); final Directory flutterDir = Directory.systemTemp.createTempSync('flutter'); final Directory flutterDirDevTools = new Directory(path.join(flutterDir.path, 'dev', 'tools')); -final String pubPath = - path.join(path.dirname(Platform.resolvedExecutable), Platform.isWindows ? 'pub.bat' : 'pub'); +final String pubPath = path.join(path.dirname(Platform.resolvedExecutable), + Platform.isWindows ? 'pub.bat' : 'pub'); final RegExp quotables = new RegExp(r'[ "\r\n\$]'); From e7b3985eaff115d78c348562eb1f49b7c3f61eba Mon Sep 17 00:00:00 2001 From: Janice Collins Date: Mon, 11 Dec 2017 15:53:53 -0800 Subject: [PATCH 09/11] Use grinder library for finding sdk binaries --- tool/grind.dart | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/tool/grind.dart b/tool/grind.dart index 46dd807ed5..b1720242c6 100644 --- a/tool/grind.dart +++ b/tool/grind.dart @@ -18,9 +18,6 @@ final Directory sdkDocsDir = Directory.systemTemp.createTempSync('sdkdocs'); final Directory flutterDir = Directory.systemTemp.createTempSync('flutter'); final Directory flutterDirDevTools = new Directory(path.join(flutterDir.path, 'dev', 'tools')); -final String pubPath = path.join(path.dirname(Platform.resolvedExecutable), - Platform.isWindows ? 'pub.bat' : 'pub'); - final RegExp quotables = new RegExp(r'[ "\r\n\$]'); // from flutter:dev/tools/dartdoc.dart, modified @@ -109,8 +106,17 @@ class _SubprocessLauncher { } @Task('Analyze dartdoc to ensure there are no errors and warnings') -analyze() { - Analyzer.analyze(['bin', 'lib', 'test', 'tool'], fatalWarnings: true); +analyze() async { + await new _SubprocessLauncher('analyze').runStreamed( + sdkBin('dartanalyzer'), + [ + '--fatal-warnings', + 'bin', + 'lib', + 'test', + 'tool', + ], + ); } @Task('analyze, test, and self-test dartdoc') @@ -136,7 +142,7 @@ Future buildSdkDocs() async { Future serveSdkDocs() async { log('launching dhttpd on port 8000 for SDK'); var launcher = new _SubprocessLauncher('serve-sdk-docs'); - await launcher.runStreamed(pubPath, [ + await launcher.runStreamed(sdkBin('pub'), [ 'run', 'dhttpd', '--port', @@ -151,8 +157,8 @@ Future serveSdkDocs() async { Future serveFlutterDocs() async { log('launching dhttpd on port 8001 for Flutter'); var launcher = new _SubprocessLauncher('serve-flutter-docs'); - await launcher.runStreamed(pubPath, ['get']); - await launcher.runStreamed(pubPath, [ + await launcher.runStreamed(sdkBin('pub'), ['get']); + await launcher.runStreamed(sdkBin('pub'), [ 'run', 'dhttpd', '--port', @@ -311,7 +317,8 @@ publish() async { test() async { // `pub run test` is a bit slower than running an `test_all.dart` script // But it provides more useful output in the case of failures. - await new _SubprocessLauncher('test').runStreamed(pubPath, ['run', 'test']); + await new _SubprocessLauncher('test') + .runStreamed(sdkBin('pub'), ['run', 'test']); } @Task('Generate docs for dartdoc') @@ -329,7 +336,7 @@ updateTestPackageDocs() async { var testPackageDocs = new Directory(path.join('testing', 'test_package_docs')); var testPackage = new Directory(path.join('testing', 'test_package')); - await launcher.runStreamed(pubPath, ['get'], + await launcher.runStreamed(sdkBin('pub'), ['get'], workingDirectory: testPackage.path); delete(testPackageDocs); // This must be synced with ../test/compare_output_test.dart's From 538c08030f6fc1b45230283bfe9a8ce0376ed395 Mon Sep 17 00:00:00 2001 From: Janice Collins Date: Tue, 12 Dec 2017 09:57:18 -0800 Subject: [PATCH 10/11] Review comments --- tool/grind.dart | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/tool/grind.dart b/tool/grind.dart index b1720242c6..a7a90d02ea 100644 --- a/tool/grind.dart +++ b/tool/grind.dart @@ -13,13 +13,32 @@ import 'package:yaml/yaml.dart' as yaml; main([List args]) => grind(args); -final Directory dartdocDocsDir = Directory.systemTemp.createTempSync('dartdoc'); -final Directory sdkDocsDir = Directory.systemTemp.createTempSync('sdkdocs'); -final Directory flutterDir = Directory.systemTemp.createTempSync('flutter'); +Directory _dartdocDocsDir; +Directory get dartdocDocsDir { + if (_dartdocDocsDir == null) + _dartdocDocsDir = Directory.systemTemp.createTempSync('dartdoc'); + return _dartdocDocsDir; +} + +Directory _sdkDocsDir; +Directory get sdkDocsDir { + if (_dartdocDocsDir == null) + _sdkDocsDir = Directory.systemTemp.createTempSync('sdkdocs'); + return _sdkDocsDir; +} + +Directory _flutterDir; +Directory get flutterDir { + if (_flutterDir == null) { + _flutterDir = Directory.systemTemp.createTempSync('flutter'); + } + return _flutterDir; +} + final Directory flutterDirDevTools = new Directory(path.join(flutterDir.path, 'dev', 'tools')); -final RegExp quotables = new RegExp(r'[ "\r\n\$]'); +final RegExp quotables = new RegExp(r'[ "\r\n\$]'); // from flutter:dev/tools/dartdoc.dart, modified void _printStream(Stream> stream, Stdout output, {String prefix: ''}) { From 3a02ad0375ddbb610b3baa451d156a7273506f1b Mon Sep 17 00:00:00 2001 From: Janice Collins Date: Tue, 12 Dec 2017 10:17:08 -0800 Subject: [PATCH 11/11] Fix error in caching --- tool/grind.dart | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tool/grind.dart b/tool/grind.dart index a7a90d02ea..3601561f9c 100644 --- a/tool/grind.dart +++ b/tool/grind.dart @@ -15,15 +15,17 @@ main([List args]) => grind(args); Directory _dartdocDocsDir; Directory get dartdocDocsDir { - if (_dartdocDocsDir == null) + if (_dartdocDocsDir == null) { _dartdocDocsDir = Directory.systemTemp.createTempSync('dartdoc'); + } return _dartdocDocsDir; } Directory _sdkDocsDir; Directory get sdkDocsDir { - if (_dartdocDocsDir == null) + if (_sdkDocsDir == null) { _sdkDocsDir = Directory.systemTemp.createTempSync('sdkdocs'); + } return _sdkDocsDir; }