Refactor search ranking (#3424)

Author: srawlins

lib/resources/docs.dart.js

@@ -43,6 +43,8 @@ class DartdocGeneratorBackendOptions implements TemplateOptions {
 
   final String? resourcesDir;
 
+  final List<String> packageOrder;
+
   DartdocGeneratorBackendOptions.fromContext(
       DartdocGeneratorOptionContext context)
       : relCanonicalPrefix = context.relCanonicalPrefix,
@@ -53,7 +55,8 @@ class DartdocGeneratorBackendOptions implements TemplateOptions {
         customHeaderContent = context.header,
         customFooterContent = context.footer,
         customInnerFooterText = context.footerText,
-        resourcesDir = context.resourcesDir;
+        resourcesDir = context.resourcesDir,
+        packageOrder = context.packageOrder;
 }
 
 /// An interface for classes which are responsible for outputing the generated
@@ -167,7 +170,7 @@ abstract class GeneratorBackendBase implements GeneratorBackend {
   @override
   void generateSearchIndex(List<Indexable> indexedElements) {
     var json = generator_util.generateSearchIndexJson(
-        indexedElements, options.prettyIndexJson);
+        indexedElements, options.prettyIndexJson, options.packageOrder);
     if (!options.useBaseHref) {
       json = json.replaceAll(htmlBasePlaceholder, '');
     }

lib/resources/docs.dart.js.map

@@ -43,12 +43,13 @@ String removeHtmlTags(String? input) {
   return parsedString;
 }
 
-String generateSearchIndexJson(
-    Iterable<Indexable> indexedElements, bool pretty) {
+String generateSearchIndexJson(Iterable<Indexable> indexedElements, bool pretty,
+    List<String> packageOrder) {
   final indexItems = [
+    {packageOrderKey: packageOrder},
     for (final indexable
         in indexedElements.sorted(_compareElementRepresentations))
-      <String, Object?>{
+      {
         'name': indexable.name,
         'qualifiedName': indexable.fullyQualifiedName,
         'href': indexable.href,
@@ -72,6 +73,9 @@ String generateSearchIndexJson(
   return encoder.convert(indexItems);
 }
 
+/// The key used in the `index.json` file used to specify the package order.
+const packageOrderKey = '__PACKAGE_ORDER__';
+
 // Compares two elements, first by fully qualified name, then by kind.
 int _compareElementRepresentations<T extends Indexable>(T a, T b) {
   final value = compareNatural(a.fullyQualifiedName, b.fullyQualifiedName);

lib/src/generator/generator_backend.dart

@@ -0,0 +1,214 @@
+// Copyright (c) 2023, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:convert';
+
+import 'package:dartdoc/src/generator/generator_utils.dart';
+import 'package:meta/meta.dart';
+
+enum _MatchPosition {
+  isExactly,
+  startsWith,
+  contains;
+
+  int operator -(_MatchPosition other) => index - other.index;
+}
+
+class Index {
+  final List<String> packageOrder;
+  final List<IndexItem> index;
+
+  @visibleForTesting
+  Index(this.packageOrder, this.index);
+
+  factory Index.fromJson(String text) {
+    var jsonIndex = (jsonDecode(text) as List).cast<Map<String, Object>>();
+    var indexList = <IndexItem>[];
+    var packageOrder = <String>[];
+    for (var entry in jsonIndex) {
+      if (entry.containsKey(packageOrderKey)) {
+        packageOrder.addAll(entry[packageOrderKey] as List<String>);
+      } else {
+        indexList.add(IndexItem.fromMap(entry));
+      }
+    }
+    return Index(packageOrder, indexList);
+  }
+
+  int packageOrderPosition(String packageName) {
+    if (packageOrder.isEmpty) return 0;
+    var index = packageOrder.indexOf(packageName);
+    return index == -1 ? packageOrder.length : index;
+  }
+
+  List<IndexItem> find(String rawQuery) {
+    if (rawQuery.isEmpty) {
+      return [];
+    }
+
+    var query = rawQuery.toLowerCase();
+    var allMatches = <({IndexItem item, _MatchPosition matchPosition})>[];
+
+    for (var item in index) {
+      void score(_MatchPosition matchPosition) {
+        allMatches.add((item: item, matchPosition: matchPosition));
+      }
+
+      var lowerName = item.name.toLowerCase();
+      var lowerQualifiedName = item.qualifiedName.toLowerCase();
+
+      if (lowerName == query ||
+          lowerQualifiedName == query ||
+          lowerName == 'dart:$query') {
+        score(_MatchPosition.isExactly);
+      } else if (query.length > 1) {
+        if (lowerName.startsWith(query) ||
+            lowerQualifiedName.startsWith(query)) {
+          score(_MatchPosition.startsWith);
+        } else if (lowerName.contains(query) ||
+            lowerQualifiedName.contains(query)) {
+          score(_MatchPosition.contains);
+        }
+      }
+    }
+
+    allMatches.sort((a, b) {
+      // Exact match vs substring is king. If the user has typed the whole term
+      // they are searching for, but it isn't at the top, they cannot type any
+      // more to try and find it.
+      var comparison = a.matchPosition - b.matchPosition;
+      if (comparison != 0) {
+        return comparison;
+      }
+
+      // Prefer packages higher in the package order.
+      comparison = packageOrderPosition(a.item.packageName) -
+          packageOrderPosition(b.item.packageName);
+      if (comparison != 0) {
+        return comparison;
+      }
+
+      // Prefer top-level elements to library members to class (etc.) members.
+      comparison = a.item._scope - b.item._scope;
+      if (comparison != 0) {
+        return comparison;
+      }
+
+      // Prefer non-overrides to overrides.
+      comparison = a.item.overriddenDepth - b.item.overriddenDepth;
+      if (comparison != 0) {
+        return comparison;
+      }
+
+      // Prefer shorter names to longer ones.
+      return a.item.name.length - b.item.name.length;
+    });
+
+    return allMatches.map((match) => match.item).toList();
+  }
+}
+
+class IndexItem {
+  final String name;
+  final String qualifiedName;
+
+  // TODO(srawlins): Store the index of the package in package order instead of
+  // this String. The Strings bloat the `index.json` file and keeping duplicate
+  // parsed Strings in memory is expensive.
+  final String packageName;
+  final String type;
+  final String? href;
+  final int overriddenDepth;
+  final String? desc;
+  final EnclosedBy? enclosedBy;
+
+  IndexItem._({
+    required this.name,
+    required this.qualifiedName,
+    required this.packageName,
+    required this.type,
+    required this.desc,
+    required this.href,
+    required this.overriddenDepth,
+    required this.enclosedBy,
+  });
+
+  // Example Map structure:
+  //
+  // ```dart
+  // {
+  //   "name":"dartdoc",
+  //   "qualifiedName":"dartdoc.Dartdoc",
+  //   "href":"dartdoc/Dartdoc-class.html",
+  //   "type":"class",
+  //   "overriddenDepth":0,
+  //   "packageName":"dartdoc"
+  //   ["enclosedBy":{"name":"dartdoc","type":"library"}]
+  // }
+  // ```
+  factory IndexItem.fromMap(Map<String, dynamic> data) {
+    // Note that this map also contains 'packageName', but we're not currently
+    // using that info.
+
+    EnclosedBy? enclosedBy;
+    if (data['enclosedBy'] != null) {
+      final map = data['enclosedBy'] as Map<String, Object>;
+      enclosedBy = EnclosedBy._(
+          name: map['name'] as String,
+          type: map['type'] as String,
+          href: map['href'] as String);
+    }
+
+    return IndexItem._(
+      name: data['name'],
+      qualifiedName: data['qualifiedName'],
+      packageName: data['packageName'],
+      href: data['href'],
+      type: data['type'],
+      overriddenDepth: (data['overriddenDepth'] as int?) ?? 0,
+      desc: data['desc'],
+      enclosedBy: enclosedBy,
+    );
+  }
+
+  /// The "scope" of a search item which may affect ranking.
+  ///
+  /// This is not the lexical scope of identifiers in Dart code, but similar in a
+  /// very loose sense. We define 4 "scopes":
+  ///
+  /// * 0: root- and package-level items
+  /// * 1: library members
+  /// * 2: container members
+  /// * 3: unknown (shouldn't be used but present for completeness)
+  // TODO(srawlins): Test and confirm that top-level functions, variables, and
+  // constants are ranked appropriately.
+  int get _scope =>
+      const {
+        'topic': 0,
+        'library': 0,
+        'class': 1,
+        'enum': 1,
+        'mixin': 1,
+        'extension': 1,
+        'typedef': 1,
+        'function': 2,
+        'method': 2,
+        'accessor': 2,
+        'operator': 2,
+        'constant': 2,
+        'property': 2,
+        'constructor': 2,
+      }[type] ??
+      3;
+}
+
+class EnclosedBy {
+  final String name;
+  final String type;
+  final String href;
+
+  // Built from JSON structure:
+  // ["enclosedBy":{"name":"Accessor","type":"class","href":"link"}]
+  EnclosedBy._({required this.name, required this.type, required this.href});
+}