diff --git a/app/bin/service/search.dart b/app/bin/service/search.dart index b19cb49f35..a9cab76a96 100644 --- a/app/bin/service/search.dart +++ b/app/bin/service/search.dart @@ -14,6 +14,8 @@ import 'package:logging/logging.dart'; import 'package:pub_dartlang_org/shared/analyzer_client.dart'; import 'package:pub_dartlang_org/shared/analyzer_memcache.dart'; import 'package:pub_dartlang_org/shared/configuration.dart'; +import 'package:pub_dartlang_org/shared/dartdoc_client.dart'; +import 'package:pub_dartlang_org/shared/dartdoc_memcache.dart'; import 'package:pub_dartlang_org/shared/handler_helpers.dart'; import 'package:pub_dartlang_org/shared/popularity_storage.dart'; import 'package:pub_dartlang_org/shared/scheduler_stats.dart'; @@ -55,6 +57,11 @@ void _main(FrontendEntryMessage message) { registerAnalyzerClient(analyzerClient); registerScopeExitCallback(analyzerClient.close); + registerDartdocMemcache(new DartdocMemcache(memcacheService)); + final DartdocClient dartdocClient = new DartdocClient(); + registerDartdocClient(dartdocClient); + registerScopeExitCallback(dartdocClient.close); + registerSearchBackend(new SearchBackend(db.dbService)); final Bucket snapshotBucket = await getOrCreateBucket( diff --git a/app/lib/search/backend.dart b/app/lib/search/backend.dart index 3f511b00e5..29dc0153f6 100644 --- a/app/lib/search/backend.dart +++ b/app/lib/search/backend.dart @@ -17,6 +17,7 @@ import 'package:json_annotation/json_annotation.dart'; import '../frontend/model_properties.dart'; import '../frontend/models.dart'; import '../shared/analyzer_client.dart'; +import '../shared/dartdoc_client.dart'; import '../shared/popularity_storage.dart'; import '../shared/search_service.dart'; import '../shared/utils.dart'; @@ -68,10 +69,15 @@ class SearchBackend { versionList.where((pv) => pv != null), key: (pv) => (pv as PackageVersion).package); + final indexJsonFutures = Future.wait(packages.map( + (p) => dartdocClient.getContentBytes(p.name, 'latest', 'index.json'))); + final List analysisViews = await analyzerClient.getAnalysisViews(packages.map((p) => p == null ? null : new AnalysisKey(p.name, p.latestVersion))); + final indexJsonContents = await indexJsonFutures; + final List results = new List(packages.length); for (int i = 0; i < packages.length; i++) { final Package p = packages[i]; @@ -82,6 +88,9 @@ class SearchBackend { final analysisView = analysisViews[i]; final double popularity = popularityStorage.lookup(pv.package) ?? 0.0; + final List indexJsonContent = indexJsonContents[i]; + final publicApiSymbols = _publicApisFromIndexJson(indexJsonContent); + results[i] = new PackageDocument( package: pv.package, version: p.latestVersion, @@ -97,6 +106,7 @@ class SearchBackend { maintenance: analysisView.maintenanceScore, dependencies: _buildDependencies(analysisView), emails: _buildEmails(p, pv), + publicApiSymbols: publicApiSymbols, timestamp: new DateTime.now().toUtc(), ); } @@ -121,6 +131,21 @@ class SearchBackend { } return emails.toList()..sort(); } + + List _publicApisFromIndexJson(List bytes) { + if (bytes == null) return null; + try { + final set = new Set(); + final list = json.decode(utf8.decode(bytes)); + for (Map map in list) { + set.add(map['name']); + } + return set.toList()..sort(); + } catch (e, st) { + _logger.warning('Parsing dartdoc index.json failed.', e, st); + } + return null; + } } class SnapshotStorage { diff --git a/app/lib/search/index_simple.dart b/app/lib/search/index_simple.dart index b1d254f4e0..4e947057b9 100644 --- a/app/lib/search/index_simple.dart +++ b/app/lib/search/index_simple.dart @@ -28,6 +28,7 @@ class SimplePackageIndex implements PackageIndex { final TokenIndex _nameIndex = new TokenIndex(minLength: 2); final TokenIndex _descrIndex = new TokenIndex(minLength: 3); final TokenIndex _readmeIndex = new TokenIndex(minLength: 3); + final TokenIndex _apiIndex = new TokenIndex(minLength: 3); final StringInternPool _internPool = new StringInternPool(); DateTime _lastUpdated; bool _isReady = false; @@ -73,6 +74,7 @@ class SimplePackageIndex implements PackageIndex { _nameIndex.add(doc.package, doc.package); _descrIndex.add(doc.package, doc.description); _readmeIndex.add(doc.package, doc.readme); + _apiIndex.add(doc.package, doc.publicApiSymbols?.join(' ')); final String allText = [doc.package, doc.description, doc.readme] .where((s) => s != null) .join(' '); @@ -93,6 +95,7 @@ class SimplePackageIndex implements PackageIndex { _nameIndex.remove(package); _descrIndex.remove(package); _readmeIndex.remove(package); + _apiIndex.remove(package); _normalizedPackageText.remove(package); } @@ -280,12 +283,18 @@ class SimplePackageIndex implements PackageIndex { final nameTokens = _nameIndex.lookupTokens(word); final descrTokens = _descrIndex.lookupTokens(word); final readmeTokens = _readmeIndex.lookupTokens(word); - - final maxTokenLength = math.max(nameTokens.maxLength, - math.max(descrTokens.maxLength, readmeTokens.maxLength)); + final apiTokens = _apiIndex.lookupTokens(word); + + final maxTokenLength = [ + nameTokens.maxLength, + descrTokens.maxLength, + readmeTokens.maxLength, + apiTokens.maxLength + ].fold(0, math.max); nameTokens.removeShortTokens(maxTokenLength); descrTokens.removeShortTokens(maxTokenLength); readmeTokens.removeShortTokens(maxTokenLength); + apiTokens.removeShortTokens(maxTokenLength); final name = new Score(_nameIndex.scoreDocs(nameTokens, weight: 1.00, wordCount: wordCount)); @@ -293,7 +302,9 @@ class SimplePackageIndex implements PackageIndex { weight: 0.95, wordCount: wordCount)); final readme = new Score(_readmeIndex.scoreDocs(readmeTokens, weight: 0.90, wordCount: wordCount)); - return Score.max([name, descr, readme]).removeLowValues( + final api = new Score( + _apiIndex.scoreDocs(apiTokens, weight: 0.80, wordCount: wordCount)); + return Score.max([name, descr, readme, api]).removeLowValues( fraction: 0.01, minValue: 0.001); }).toList(); Score score = Score.multiply(wordScores); diff --git a/app/lib/shared/dartdoc_client.dart b/app/lib/shared/dartdoc_client.dart index efbfea070f..ff2938b9c7 100644 --- a/app/lib/shared/dartdoc_client.dart +++ b/app/lib/shared/dartdoc_client.dart @@ -7,6 +7,7 @@ import 'dart:async'; import 'package:gcloud/service_scope.dart' as ss; import 'package:http/http.dart' as http; import 'package:logging/logging.dart'; +import 'package:path/path.dart' as p; import 'package:pool/pool.dart'; import '../dartdoc/dartdoc_runner.dart' show statusFilePath; @@ -15,6 +16,7 @@ import '../dartdoc/models.dart' show DartdocEntry; import 'configuration.dart'; import 'dartdoc_memcache.dart'; import 'notification.dart' show notifyService; +import 'utils.dart' show getUrlWithRetry; export '../dartdoc/models.dart' show DartdocEntry; @@ -57,25 +59,33 @@ class DartdocClient { _client.close(); } - Future _getEntry(String package, String version) async { - final cachedContent = - await dartdocMemcache?.getEntryBytes(package, version, true); - if (cachedContent != null) { - return new DartdocEntry.fromBytes(cachedContent); - } - final url = - '$_dartdocServiceHttpHostPort/documentation/$package/$version/$statusFilePath'; + Future> getContentBytes( + String package, String version, String relativePath) async { + final url = p.join(_dartdocServiceHttpHostPort, 'documentation', package, + version, relativePath); try { - final rs = await _client.get(url); + final rs = await getUrlWithRetry(_client, url); if (rs.statusCode != 200) { return null; } - await dartdocMemcache?.setEntryBytes( - package, version, true, rs.bodyBytes); - return new DartdocEntry.fromBytes(rs.bodyBytes); + return rs.bodyBytes; } catch (e) { _logger.info('Error requesting entry for: $package $version'); } return null; } + + Future _getEntry(String package, String version) async { + final cachedContent = + await dartdocMemcache?.getEntryBytes(package, version, true); + if (cachedContent != null) { + return new DartdocEntry.fromBytes(cachedContent); + } + final content = await getContentBytes(package, version, statusFilePath); + if (content != null) { + await dartdocMemcache?.setEntryBytes(package, version, true, content); + return new DartdocEntry.fromBytes(content); + } + return null; + } } diff --git a/app/lib/shared/search_service.dart b/app/lib/shared/search_service.dart index 21d25f4979..9185490dc0 100644 --- a/app/lib/shared/search_service.dart +++ b/app/lib/shared/search_service.dart @@ -58,6 +58,8 @@ class PackageDocument extends Object with _$PackageDocumentSerializerMixin { final Map dependencies; final List emails; + final List publicApiSymbols; + /// The creation timestamp of this document. final DateTime timestamp; @@ -76,6 +78,7 @@ class PackageDocument extends Object with _$PackageDocumentSerializerMixin { this.maintenance, this.dependencies, this.emails, + this.publicApiSymbols, this.timestamp, }); @@ -104,6 +107,7 @@ class PackageDocument extends Object with _$PackageDocumentSerializerMixin { value: (key) => internFn(dependencies[key]), ), emails: emails?.map(internFn)?.toList(), + publicApiSymbols: publicApiSymbols?.map(internFn)?.toList(), timestamp: timestamp, ); } diff --git a/app/lib/shared/search_service.g.dart b/app/lib/shared/search_service.g.dart index d4d5dbac89..39f2d5ae6b 100644 --- a/app/lib/shared/search_service.g.dart +++ b/app/lib/shared/search_service.g.dart @@ -35,6 +35,9 @@ PackageDocument _$PackageDocumentFromJson(Map json) => ? null : new Map.from(json['dependencies'] as Map), emails: (json['emails'] as List)?.map((e) => e as String)?.toList(), + publicApiSymbols: (json['publicApiSymbols'] as List) + ?.map((e) => e as String) + ?.toList(), timestamp: json['timestamp'] == null ? null : DateTime.parse(json['timestamp'] as String)); @@ -54,6 +57,7 @@ abstract class _$PackageDocumentSerializerMixin { double get maintenance; Map get dependencies; List get emails; + List get publicApiSymbols; DateTime get timestamp; Map toJson() => { 'package': package, @@ -70,6 +74,7 @@ abstract class _$PackageDocumentSerializerMixin { 'maintenance': maintenance, 'dependencies': dependencies, 'emails': emails, + 'publicApiSymbols': publicApiSymbols, 'timestamp': timestamp?.toIso8601String() }; } diff --git a/app/test/frontend/handlers_test_utils.dart b/app/test/frontend/handlers_test_utils.dart index d9b339237f..3884b0ac25 100644 --- a/app/test/frontend/handlers_test_utils.dart +++ b/app/test/frontend/handlers_test_utils.dart @@ -319,4 +319,10 @@ class DartdocClientMock implements DartdocClient { @override Future close() async {} + + @override + Future> getContentBytes( + String package, String version, String relativePath) async { + return null; + } } diff --git a/app/test/search/index_api_test.dart b/app/test/search/index_api_test.dart new file mode 100644 index 0000000000..fc2c25775a --- /dev/null +++ b/app/test/search/index_api_test.dart @@ -0,0 +1,118 @@ +// Copyright (c) 2018, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:convert'; + +import 'package:test/test.dart'; + +import 'package:pub_dartlang_org/search/index_simple.dart'; +import 'package:pub_dartlang_org/search/text_utils.dart'; +import 'package:pub_dartlang_org/shared/search_service.dart'; + +void main() { + group('api', () { + SimplePackageIndex index; + + setUpAll(() async { + index = new SimplePackageIndex(); + await index.addPackage(new PackageDocument( + package: 'foo', + version: '1.0.0', + description: compactDescription('Yet another web framework.'), + publicApiSymbols: [ + 'generateWebPage', + 'WebPageGenerator', + ], + )); + await index.addPackage(new PackageDocument( + package: 'other_with_api', + version: '2.0.0', + description: compactDescription('Unrelated package'), + publicApiSymbols: [ + 'foo', + 'serveWebPages', + ], + )); + await index.addPackage(new PackageDocument( + package: 'other_without_api', + version: '2.0.0', + description: compactDescription('Unrelated package'), + )); + await index.merge(); + }); + + test('foo', () async { + final PackageSearchResult result = await index + .search(new SearchQuery.parse(query: 'foo', order: SearchOrder.text)); + expect(json.decode(json.encode(result)), { + 'indexUpdated': isNotNull, + 'totalCount': 2, + 'packages': [ + { + 'package': 'foo', + 'score': closeTo(0.986, 0.001), // finds package name + }, + { + 'package': 'other_with_api', + 'score': closeTo(0.772, 0.001), // finds foo method + }, + // should not contain `other_without_api` + ], + }); + }); + + test('server', () async { + final PackageSearchResult result = await index.search( + new SearchQuery.parse(query: 'server', order: SearchOrder.text)); + expect(json.decode(json.encode(result)), { + 'indexUpdated': isNotNull, + 'totalCount': 1, + 'packages': [ + { + 'package': 'other_with_api', + 'score': closeTo(0.772, 0.001), // find serveWebPages + }, + // should not contain `other_without_api` + ], + }); + }); + + test('page generator', () async { + final PackageSearchResult result = await index.search( + new SearchQuery.parse( + query: 'page generator', order: SearchOrder.text)); + expect(json.decode(json.encode(result)), { + 'indexUpdated': isNotNull, + 'totalCount': 1, + 'packages': [ + { + 'package': 'foo', + 'score': closeTo(0.614, 0.001), // find WebPageGenerator + }, + // should not contain `other_without_api` + ], + }); + }); + + test('web page', () async { + final PackageSearchResult result = await index.search( + new SearchQuery.parse(query: 'web page', order: SearchOrder.text)); + expect(json.decode(json.encode(result)), { + 'indexUpdated': isNotNull, + 'totalCount': 2, + 'packages': [ + { + 'package': 'foo', + 'score': closeTo(0.731, 0.001), // find WebPageGenerator + }, + { + 'package': 'other_with_api', + 'score': closeTo(0.617, 0.001), // find serveWebPages + }, + // should not contain `other_without_api` + ], + }); + }); + }); +}