Use extracted dartdoc texts in search. (#1428)

* Use extracted dartdoc texts in search.

* Fix code review notes: wrapping and log level.

Author: isoos

app/lib/search/backend.dart

@@ -14,6 +14,7 @@ import 'package:gcloud/storage.dart';
 import 'package:gcloud/service_scope.dart' as ss;
 import 'package:json_annotation/json_annotation.dart';
 
+import '../dartdoc/pub_dartdoc_data.dart';
 import '../frontend/model_properties.dart';
 import '../frontend/models.dart';
 import '../shared/analyzer_client.dart';
@@ -70,15 +71,15 @@ class SearchBackend {
         versionList.where((pv) => pv != null),
         key: (pv) => (pv as PackageVersion).package);
 
-    final indexJsonFutures = Future.wait(packages.map((p) =>
-        dartdocClient.getContentBytes(p.name, 'latest', 'index.json',
+    final pubDataFutures = Future.wait(packages.map((p) =>
+        dartdocClient.getContentBytes(p.name, 'latest', 'pub-data.json',
             timeout: const Duration(seconds: 10))));
 
     final List<AnalysisView> analysisViews =
         await analyzerClient.getAnalysisViews(packages.map((p) =>
             p == null ? null : new AnalysisKey(p.name, p.latestVersion)));
 
-    final indexJsonContents = await indexJsonFutures;
+    final pubDataContents = await pubDataFutures;
 
     final List<PackageDocument> results = new List(packages.length);
     for (int i = 0; i < packages.length; i++) {
@@ -90,8 +91,15 @@ class SearchBackend {
       final analysisView = analysisViews[i];
       final double popularity = popularityStorage.lookup(pv.package) ?? 0.0;
 
-      final List<int> indexJsonContent = indexJsonContents[i];
-      final apiDocPages = _apiDocPagesFromIndexJson(indexJsonContent);
+      final List<int> pubDataContent = pubDataContents[i];
+      List<ApiDocPage> apiDocPages;
+      if (pubDataContent != null) {
+        try {
+          apiDocPages = _apiDocPagesFromPubData(pubDataContent);
+        } catch (e, st) {
+          _logger.severe('Parsing pub-data.json failed.', e, st);
+        }
+      }
 
       results[i] = new PackageDocument(
         package: pv.package,
@@ -135,43 +143,56 @@ class SearchBackend {
     return emails.toList()..sort();
   }
 
-  List<ApiDocPage> _apiDocPagesFromIndexJson(List<int> bytes) {
-    if (bytes == null) return null;
-    try {
-      final list = json.decode(utf8.decode(bytes));
-
-      final pathMap = <String, String>{};
-      final symbolMap = <String, Set<String>>{};
-      for (Map map in list) {
-        final String name = map['name'];
-        final type = map['type'];
-        if (isCommonApiSymbol(name) && type != 'library') {
-          continue;
-        }
+  List<ApiDocPage> _apiDocPagesFromPubData(List<int> bytes) {
+    final decodedMap = json.decode(utf8.decode(bytes)) as Map;
+    final pubData = new PubDartdocData.fromJson(decodedMap.cast());
+
+    final nameToKindMap = <String, String>{};
+    pubData.apiElements.forEach((e) {
+      nameToKindMap[e.name] = e.kind;
+    });
 
-        final String qualifiedName = map['qualifiedName'];
-        final enclosedBy = map['enclosedBy'];
-        final enclosedByType = enclosedBy is Map ? enclosedBy['type'] : null;
-        final parentLevel = enclosedByType == 'class' ? 2 : 1;
-        final String key = qualifiedName.split('.').take(parentLevel).join('.');
+    final pathMap = <String, String>{};
+    final symbolMap = <String, Set<String>>{};
+    final docMap = <String, List<String>>{};
 
-        if (key == qualifiedName) {
-          pathMap[key] = map['href'] as String;
-        }
-        symbolMap.putIfAbsent(key, () => new Set()).add(name);
-      }
+    bool isTopLevel(String kind) => kind == 'library' || kind == 'class';
 
-      final results = pathMap.keys.map((key) {
-        final path = pathMap[key];
-        final symbols = symbolMap[key].toList()..sort();
-        return new ApiDocPage(relativePath: path, symbols: symbols);
-      }).toList();
-      results.sort((a, b) => a.relativePath.compareTo(b.relativePath));
-      return results;
-    } catch (e, st) {
-      _logger.warning('Parsing dartdoc index.json failed.', e, st);
+    void update(String key, String name, String documentation) {
+      final set = symbolMap.putIfAbsent(key, () => new Set<String>());
+      set.addAll(name.split('.'));
+
+      documentation = documentation?.trim();
+      if (documentation != null && documentation.isNotEmpty) {
+        final list = docMap.putIfAbsent(key, () => []);
+        list.add(compactReadme(documentation));
+      }
     }
-    return null;
+
+    pubData.apiElements.forEach((apiElement) {
+      if (isTopLevel(apiElement.kind)) {
+        pathMap[apiElement.name] = apiElement.href;
+        update(apiElement.name, apiElement.name, apiElement.documentation);
+      }
+
+      if (!isTopLevel(apiElement.kind) &&
+          apiElement.parent != null &&
+          isTopLevel(nameToKindMap[apiElement.parent])) {
+        update(apiElement.parent, apiElement.name, apiElement.documentation);
+      }
+    });
+
+    final results = pathMap.keys.map((key) {
+      final path = pathMap[key];
+      final symbols = symbolMap[key].toList()..sort();
+      return new ApiDocPage(
+        relativePath: path,
+        symbols: symbols,
+        textBlocks: docMap[key],
+      );
+    }).toList();
+    results.sort((a, b) => a.relativePath.compareTo(b.relativePath));
+    return results;
   }
 }
 

app/lib/search/index_simple.dart

@@ -85,8 +85,11 @@ class SimplePackageIndex implements PackageIndex {
     _descrIndex.add(doc.package, doc.description);
     _readmeIndex.add(doc.package, doc.readme);
     for (ApiDocPage page in doc.apiDocPages ?? const []) {
-      _apiDocIndex.add(
-          _apiDocPageId(doc.package, page), page.symbols?.join(' '));
+      final text = [page.symbols, page.textBlocks]
+          .where((list) => list != null && list.isNotEmpty)
+          .expand((list) => list)
+          .join(' ');
+      _apiDocIndex.add(_apiDocPageId(doc.package, page), text);
     }
     final String allText = [doc.package, doc.description, doc.readme]
         .where((s) => s != null)

app/lib/shared/search_service.dart

@@ -116,8 +116,9 @@ class PackageDocument extends Object with _$PackageDocumentSerializerMixin {
 class ApiDocPage extends Object with _$ApiDocPageSerializerMixin {
   final String relativePath;
   final List<String> symbols;
+  final List<String> textBlocks;
 
-  ApiDocPage({this.relativePath, this.symbols});
+  ApiDocPage({this.relativePath, this.symbols, this.textBlocks});
 
   factory ApiDocPage.fromJson(Map<String, dynamic> json) =>
       _$ApiDocPageFromJson(json);
@@ -126,6 +127,7 @@ class ApiDocPage extends Object with _$ApiDocPageSerializerMixin {
     return new ApiDocPage(
       relativePath: internFn(relativePath),
       symbols: symbols?.map(internFn)?.toList(),
+      textBlocks: textBlocks,
     );
   }
 }

app/lib/shared/search_service.g.dart

@@ -86,14 +86,20 @@ abstract class _$PackageDocumentSerializerMixin {
 ApiDocPage _$ApiDocPageFromJson(Map<String, dynamic> json) {
   return new ApiDocPage(
       relativePath: json['relativePath'] as String,
-      symbols: (json['symbols'] as List)?.map((e) => e as String)?.toList());
+      symbols: (json['symbols'] as List)?.map((e) => e as String)?.toList(),
+      textBlocks:
+          (json['textBlocks'] as List)?.map((e) => e as String)?.toList());
 }
 
 abstract class _$ApiDocPageSerializerMixin {
   String get relativePath;
   List<String> get symbols;
-  Map<String, dynamic> toJson() =>
-      <String, dynamic>{'relativePath': relativePath, 'symbols': symbols};
+  List<String> get textBlocks;
+  Map<String, dynamic> toJson() => <String, dynamic>{
+        'relativePath': relativePath,
+        'symbols': symbols,
+        'textBlocks': textBlocks
+      };
 }
 
 PackageSearchResult _$PackageSearchResultFromJson(Map<String, dynamic> json) {

app/test/search/api_doc_page_test.dart

@@ -27,6 +27,9 @@ void main() {
               'generateWebPage',
               'WebPageGenerator',
             ],
+            textBlocks: [
+              'Some fancy goal is described here.',
+            ],
           ),
         ],
       ));
@@ -100,7 +103,7 @@ void main() {
         'packages': [
           {
             'package': 'foo',
-            'score': closeTo(0.481, 0.001), // find WebPageGenerator
+            'score': closeTo(0.478, 0.001), // find WebPageGenerator
             'apiPages': [
               {'title': null, 'path': 'generator.html'},
             ],
@@ -119,7 +122,7 @@ void main() {
         'packages': [
           {
             'package': 'foo',
-            'score': closeTo(0.572, 0.001), // find WebPageGenerator
+            'score': closeTo(0.570, 0.001), // find WebPageGenerator
             'apiPages': [
               {'title': null, 'path': 'generator.html'},
             ],
@@ -135,5 +138,23 @@ void main() {
         ],
       });
     });
+
+    test('text block', () async {
+      final PackageSearchResult result = await index.search(
+          new SearchQuery.parse(query: 'goal fancy', order: SearchOrder.text));
+      expect(json.decode(json.encode(result)), {
+        'indexUpdated': isNotNull,
+        'totalCount': 1,
+        'packages': [
+          {
+            'package': 'foo',
+            'score': closeTo(0.624, 0.001),
+            'apiPages': [
+              {'title': null, 'path': 'generator.html'},
+            ],
+          },
+        ],
+      });
+    });
   });
 }