From cf01f54c68873b5e2bdfef31a232d92ea2d5de64 Mon Sep 17 00:00:00 2001 From: Istvan Soos Date: Tue, 20 May 2025 19:10:04 +0200 Subject: [PATCH] Merging separate Dart and Flutter SDK indexes into a single index. --- app/bin/tools/sdk_search_benchmark.dart | 4 +- app/lib/search/backend.dart | 6 +- app/lib/search/dart_sdk_mem_index.dart | 38 --------- app/lib/search/flutter_sdk_mem_index.dart | 61 --------------- app/lib/search/result_combiner.dart | 15 ++-- app/lib/search/sdk_mem_index.dart | 77 +++++++++++-------- app/lib/service/entrypoint/search_index.dart | 6 +- app/test/search/backend_test.dart | 6 +- .../search/dartdoc_index_parsing_test.dart | 5 +- app/test/search/result_combiner_test.dart | 9 +-- app/test/search/sdk_mem_index_test.dart | 25 +++--- 11 files changed, 84 insertions(+), 168 deletions(-) delete mode 100644 app/lib/search/dart_sdk_mem_index.dart delete mode 100644 app/lib/search/flutter_sdk_mem_index.dart diff --git a/app/bin/tools/sdk_search_benchmark.dart b/app/bin/tools/sdk_search_benchmark.dart index db87632c41..4b3c439bba 100644 --- a/app/bin/tools/sdk_search_benchmark.dart +++ b/app/bin/tools/sdk_search_benchmark.dart @@ -2,11 +2,11 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. -import 'package:pub_dev/search/flutter_sdk_mem_index.dart'; +import 'package:pub_dev/search/sdk_mem_index.dart'; /// Loads a Dart SDK search snapshot and executes queries on it, benchmarking their total time to complete. Future main() async { - final index = await createFlutterSdkMemIndex(); + final index = await createSdkMemIndex(); // NOTE: please add more queries to this list, especially if there is a performance bottleneck. final queries = [ diff --git a/app/lib/search/backend.dart b/app/lib/search/backend.dart index 425b4c3ced..370cf68cd0 100644 --- a/app/lib/search/backend.dart +++ b/app/lib/search/backend.dart @@ -42,10 +42,9 @@ import '../task/backend.dart'; import '../task/global_lock.dart'; import '../task/models.dart'; -import 'dart_sdk_mem_index.dart'; -import 'flutter_sdk_mem_index.dart'; import 'models.dart'; import 'result_combiner.dart'; +import 'sdk_mem_index.dart'; import 'search_client.dart'; import 'search_service.dart'; import 'text_utils.dart'; @@ -623,8 +622,7 @@ class _CombinedSearchIndex implements SearchIndex { PackageSearchResult search(ServiceSearchQuery query) { final combiner = SearchResultCombiner( primaryIndex: _packageIndexHolder._index, - dartSdkMemIndex: dartSdkMemIndex, - flutterSdkMemIndex: flutterSdkMemIndex, + sdkMemIndex: sdkMemIndex, ); return combiner.search(query); } diff --git a/app/lib/search/dart_sdk_mem_index.dart b/app/lib/search/dart_sdk_mem_index.dart deleted file mode 100644 index 06bbc9c0e0..0000000000 --- a/app/lib/search/dart_sdk_mem_index.dart +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright (c) 2021, the Dart project authors. Please see the AUTHORS file -// for details. All rights reserved. Use of this source code is governed by a -// BSD-style license that can be found in the LICENSE file. - -import 'package:gcloud/service_scope.dart' as ss; -import 'package:logging/logging.dart'; - -import 'backend.dart'; -import 'sdk_mem_index.dart'; - -final _logger = Logger('search.dart_sdk_mem_index'); - -/// Sets the Dart SDK in-memory index. -void registerDartSdkMemIndex(SdkMemIndex? index) { - if (index != null) { - ss.register(#_dartSdkMemIndex, index); - } -} - -/// The active Dart SDK in-memory index. -SdkMemIndex? get dartSdkMemIndex => - ss.lookup(#_dartSdkMemIndex) as SdkMemIndex?; - -/// Tries to load Dart SDK's dartdoc `index.json` and build -/// a search index from it. -/// -/// Returns `null` when the loading of `index.json` failed, or when there -/// was an error parsing the file or building the index. -Future createDartSdkMemIndex() async { - try { - final content = - await loadOrFetchSdkIndexJsonAsString(SdkMemIndex.dartSdkIndexJsonUri); - return SdkMemIndex.dart(index: DartdocIndex.parseJsonText(content)); - } catch (e, st) { - _logger.warning('Unable to load Dart SDK index.', e, st); - return null; - } -} diff --git a/app/lib/search/flutter_sdk_mem_index.dart b/app/lib/search/flutter_sdk_mem_index.dart deleted file mode 100644 index 7f0e405ed3..0000000000 --- a/app/lib/search/flutter_sdk_mem_index.dart +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright (c) 2021, the Dart project authors. Please see the AUTHORS file -// for details. All rights reserved. Use of this source code is governed by a -// BSD-style license that can be found in the LICENSE file. - -import 'package:gcloud/service_scope.dart' as ss; -import 'package:logging/logging.dart'; - -import 'backend.dart'; -import 'sdk_mem_index.dart'; - -/// The index.json file contains overlap with the Dart SDK and also repeats -/// regular packages. The selected libraries are unique to the index.json. -/// -/// TODO: try to find a way to derive this list automatically. -const flutterSdkAllowedLibraries = { - 'dart:ui', - 'animation', - 'cupertino', - 'foundation', - 'gestures', - 'material', - 'painting', - 'physics', - 'rendering', - 'scheduler', - 'semantics', - 'services', - 'widgets', - 'flutter_test', - 'flutter_driver', - 'flutter_driver_extension', - 'flutter_web_plugins', -}; - -final _logger = Logger('search.flutter_sdk_mem_index'); - -/// Sets the Flutter SDK in-memory index. -void registerFlutterSdkMemIndex(SdkMemIndex? index) { - if (index != null) { - ss.register(#_flutterSdkMemIndex, index); - } -} - -/// The active Flutter SDK in-memory index. -SdkMemIndex? get flutterSdkMemIndex => - ss.lookup(#_flutterSdkMemIndex) as SdkMemIndex?; - -/// Creates Flutter SDK in-memory index that fetches `index.json` from -/// api.flutter.dev and returns search results based on [SdkMemIndex]. -Future createFlutterSdkMemIndex() async { - try { - final content = await loadOrFetchSdkIndexJsonAsString( - SdkMemIndex.flutterSdkIndexJsonUri); - final index = - SdkMemIndex.flutter(index: DartdocIndex.parseJsonText(content)); - return index; - } catch (e, st) { - _logger.warning('Unable to load Flutter SDK index.', e, st); - return null; - } -} diff --git a/app/lib/search/result_combiner.dart b/app/lib/search/result_combiner.dart index e6b94c3c9e..49a12978df 100644 --- a/app/lib/search/result_combiner.dart +++ b/app/lib/search/result_combiner.dart @@ -13,13 +13,11 @@ import 'search_service.dart'; /// SDK index. class SearchResultCombiner { final InMemoryPackageIndex primaryIndex; - final SdkMemIndex? dartSdkMemIndex; - final SdkMemIndex? flutterSdkMemIndex; + final SdkMemIndex? sdkMemIndex; SearchResultCombiner({ required this.primaryIndex, - required this.dartSdkMemIndex, - required this.flutterSdkMemIndex, + required this.sdkMemIndex, }); PackageSearchResult search(ServiceSearchQuery query) { @@ -30,11 +28,10 @@ class SearchResultCombiner { final queryFlutterSdk = query.tagsPredicate.hasNoTagPrefix('sdk:') || query.tagsPredicate.hasTag(SdkTag.sdkFlutter); - final sdkLibraryHits = [ - ...?dartSdkMemIndex?.search(query.query!, limit: 2), - if (queryFlutterSdk) - ...?flutterSdkMemIndex?.search(query.query!, limit: 2), - ]; + final sdkLibraryHits = sdkMemIndex + ?.search(query.query!, limit: 2, skipFlutter: !queryFlutterSdk) + .toList() ?? + []; if (sdkLibraryHits.isNotEmpty) { // Do not display low SDK scores if the package hits are more relevant on the page. // diff --git a/app/lib/search/sdk_mem_index.dart b/app/lib/search/sdk_mem_index.dart index b3d28331ea..172ed782b3 100644 --- a/app/lib/search/sdk_mem_index.dart +++ b/app/lib/search/sdk_mem_index.dart @@ -4,17 +4,29 @@ import 'dart:math'; +import 'package:gcloud/service_scope.dart' as ss; +import 'package:logging/logging.dart'; import 'package:meta/meta.dart'; // ignore: implementation_imports import 'package:pana/src/dartdoc/dartdoc_index.dart'; import 'package:path/path.dart' as p; -import 'package:pub_dev/search/flutter_sdk_mem_index.dart'; +import 'package:pub_dev/search/backend.dart'; import 'search_service.dart'; import 'token_index.dart'; export 'package:pana/src/dartdoc/dartdoc_index.dart'; +/// Sets the SDK in-memory index. +void registerSdkMemIndex(SdkMemIndex? index) { + if (index != null) { + ss.register(#_sdkMemIndex, index); + } +} + +/// The active SDK in-memory index. +SdkMemIndex? get sdkMemIndex => ss.lookup(#_sdkMemIndex) as SdkMemIndex?; + /// Results from these libraries are ranked with lower score and /// will be displayed only if the query has the library name, or /// there are not other results that could match the query. @@ -30,48 +42,54 @@ const _defaultApiPageDirWeights = { 'material/Icons': 0.25, }; +final _logger = Logger('search.dart_sdk_mem_index'); +final _dartUri = Uri.parse('https://api.dart.dev/stable/latest/'); +final _flutterUri = Uri.parse('https://api.flutter.dev/flutter/'); + +/// Tries to load Dart and Flutter SDK's dartdoc `index.json` and build +/// a search index from it. +/// +/// Returns `null` when the loading of `index.json` failed, or when there +/// was an error parsing the file or building the index. +Future createSdkMemIndex() async { + try { + final dartSdkContent = + await loadOrFetchSdkIndexJsonAsString(SdkMemIndex.dartSdkIndexJsonUri); + final flutterSdkContent = await loadOrFetchSdkIndexJsonAsString( + SdkMemIndex._flutterSdkIndexJsonUri); + return SdkMemIndex( + dartIndex: DartdocIndex.parseJsonText(dartSdkContent), + flutterIndex: DartdocIndex.parseJsonText(flutterSdkContent), + ); + } catch (e, st) { + _logger.warning('Unable to load SDK index.', e, st); + return null; + } +} + /// In-memory index for SDK library search queries. class SdkMemIndex { final _libraries = {}; final Map _apiPageDirWeights; SdkMemIndex({ - required String sdk, - required Uri baseUri, - required DartdocIndex index, - Set? allowedLibraries, + required DartdocIndex dartIndex, + required DartdocIndex flutterIndex, Map? apiPageDirWeights, }) : _apiPageDirWeights = apiPageDirWeights ?? _defaultApiPageDirWeights { - _addDartdocIndex(sdk, baseUri, index, allowedLibraries); - } - - static SdkMemIndex dart({required DartdocIndex index}) { - return SdkMemIndex( - sdk: 'dart', - baseUri: Uri.parse('https://api.dart.dev/stable/latest/'), - index: index, - ); - } - - factory SdkMemIndex.flutter({required DartdocIndex index}) { - return SdkMemIndex( - sdk: 'flutter', - baseUri: Uri.parse('https://api.flutter.dev/flutter/'), - index: index, - allowedLibraries: flutterSdkAllowedLibraries, - ); + _addDartdocIndex('dart', _dartUri, dartIndex); + _addDartdocIndex('flutter', _flutterUri, flutterIndex); } static final dartSdkIndexJsonUri = Uri.parse('https://api.dart.dev/stable/latest/index.json'); - static final flutterSdkIndexJsonUri = + static final _flutterSdkIndexJsonUri = Uri.parse('https://api.flutter.dev/flutter/index.json'); void _addDartdocIndex( String sdk, Uri baseUri, DartdocIndex index, - Set? allowedLibraries, ) { final textsPerLibrary = >{}; final baseUris = {}; @@ -81,11 +99,7 @@ class SdkMemIndex { final library = f.qualifiedName?.split('.').first; if (library == null) continue; if (f.href == null) continue; - if (allowedLibraries != null && - allowedLibraries.isNotEmpty && - !allowedLibraries.contains(library)) { - continue; - } + if (_libraries.containsKey(library)) continue; if (f.isLibrary) { baseUris[library] = baseUri.resolve(f.href!); @@ -116,6 +130,7 @@ class SdkMemIndex { List search( String query, { int? limit, + bool skipFlutter = false, }) { limit ??= 2; final words = query.split(' ').where((e) => e.isNotEmpty).toList(); @@ -123,6 +138,7 @@ class SdkMemIndex { final hits = <_Hit>[]; for (final library in _libraries.values) { + if (skipFlutter && library.isFlutter) continue; // We may reduce the rank of certain libraries, except when their name is // also part of the query. E.g. `dart:html` with `query=cursor` may be // scored lower than `query=html cursor`. @@ -204,6 +220,7 @@ class _Library { required this.tokenIndex, }); + late final isFlutter = sdk == 'flutter'; late final weight = _libraryWeights[name] ?? 1.0; late final lastNamePart = name.split(':').last; } diff --git a/app/lib/service/entrypoint/search_index.dart b/app/lib/service/entrypoint/search_index.dart index 7fd5abfe8f..f6a3a6e6f8 100644 --- a/app/lib/service/entrypoint/search_index.dart +++ b/app/lib/service/entrypoint/search_index.dart @@ -9,8 +9,7 @@ import 'dart:isolate'; import 'package:gcloud/service_scope.dart'; import 'package:logging/logging.dart'; import 'package:pub_dev/search/backend.dart'; -import 'package:pub_dev/search/dart_sdk_mem_index.dart'; -import 'package:pub_dev/search/flutter_sdk_mem_index.dart'; +import 'package:pub_dev/search/sdk_mem_index.dart'; import 'package:pub_dev/search/search_service.dart'; import 'package:pub_dev/search/updater.dart'; import 'package:pub_dev/service/entrypoint/_isolate.dart'; @@ -37,8 +36,7 @@ Future main(List args, var message) async { } await fork(() async { await servicesWrapperFn(() async { - registerDartSdkMemIndex(await createDartSdkMemIndex()); - registerFlutterSdkMemIndex(await createFlutterSdkMemIndex()); + registerSdkMemIndex(await createSdkMemIndex()); await indexUpdater.init(); final requestReceivePort = ReceivePort(); diff --git a/app/test/search/backend_test.dart b/app/test/search/backend_test.dart index 84b5c2851a..a7c878980e 100644 --- a/app/test/search/backend_test.dart +++ b/app/test/search/backend_test.dart @@ -15,8 +15,10 @@ void main() { testWithProfile('fetch SDK library description', fn: () async { final content = await loadOrFetchSdkIndexJsonAsString( SdkMemIndex.dartSdkIndexJsonUri); - final index = - SdkMemIndex.dart(index: DartdocIndex.parseJsonText(content)); + final index = SdkMemIndex( + dartIndex: DartdocIndex.parseJsonText(content), + flutterIndex: DartdocIndex([]), + ); expect( index.getLibraryDescription('dart:async'), 'Support for asynchronous programming, with classes such as Future and Stream.', diff --git a/app/test/search/dartdoc_index_parsing_test.dart b/app/test/search/dartdoc_index_parsing_test.dart index 1ecbb2db41..1469009d4b 100644 --- a/app/test/search/dartdoc_index_parsing_test.dart +++ b/app/test/search/dartdoc_index_parsing_test.dart @@ -66,7 +66,10 @@ void main() { expect(parserWithoutFirstEntry, originalWithoutFirstEntry); // parsing into SDK index - final sdkMemIndex = SdkMemIndex.flutter(index: index); + final sdkMemIndex = SdkMemIndex( + dartIndex: DartdocIndex([]), + flutterIndex: index, + ); final rs = sdkMemIndex.search('StatelessWidget'); expect(json.decode(json.encode(rs)), [ { diff --git a/app/test/search/result_combiner_test.dart b/app/test/search/result_combiner_test.dart index fd332ac3e7..6e92b9790c 100644 --- a/app/test/search/result_combiner_test.dart +++ b/app/test/search/result_combiner_test.dart @@ -9,7 +9,6 @@ import 'package:pub_dev/search/mem_index.dart'; import 'package:pub_dev/search/result_combiner.dart'; import 'package:pub_dev/search/sdk_mem_index.dart'; import 'package:pub_dev/search/search_service.dart'; -import 'package:pub_dev/shared/versions.dart'; import 'package:test/test.dart'; void main() { @@ -30,10 +29,8 @@ void main() { ); final combiner = SearchResultCombiner( primaryIndex: primaryIndex, - dartSdkMemIndex: SdkMemIndex( - sdk: 'dart', - baseUri: Uri.parse('https://api.dart.dev/stable/$runtimeSdkVersion/'), - index: DartdocIndex.fromJsonList([ + sdkMemIndex: SdkMemIndex( + dartIndex: DartdocIndex.fromJsonList([ { 'name': 'dart:core', 'qualifiedName': 'dart:core', @@ -71,8 +68,8 @@ void main() { 'enclosedBy': {'name': 'String', 'kind': 3} }, ]), + flutterIndex: DartdocIndex([]), ), - flutterSdkMemIndex: null, ); test('non-text ranking', () async { diff --git a/app/test/search/sdk_mem_index_test.dart b/app/test/search/sdk_mem_index_test.dart index b6f2f9d208..ceac1bd2b2 100644 --- a/app/test/search/sdk_mem_index_test.dart +++ b/app/test/search/sdk_mem_index_test.dart @@ -13,9 +13,7 @@ void main() { setUpAll(() async { index = SdkMemIndex( - sdk: 'dart', - baseUri: Uri.parse('https://api.dart.dev/x/'), - index: DartdocIndex.fromJsonList([ + dartIndex: DartdocIndex.fromJsonList([ { 'name': 'dart:async', 'qualifiedName': 'dart:async', @@ -80,6 +78,7 @@ void main() { 'enclosedBy': {'name': 'dart:html.FakeIcons', 'kind': 3}, }, ]), + flutterIndex: DartdocIndex([]), ); }); @@ -92,22 +91,24 @@ void main() { 'sdk': 'dart', 'library': 'dart:async', 'description': 'async description', - 'url': 'https://api.dart.dev/x/dart-async/dart-async-library.html', + 'url': + 'https://api.dart.dev/stable/latest/dart-async/dart-async-library.html', 'score': closeTo(0.98, 0.01), 'apiPages': [ { 'path': 'dart-async/AsyncError-class.html', - 'url': 'https://api.dart.dev/x/dart-async/AsyncError-class.html' + 'url': + 'https://api.dart.dev/stable/latest/dart-async/AsyncError-class.html' }, { 'path': 'dart-async/AsyncError/AsyncError.html', 'url': - 'https://api.dart.dev/x/dart-async/AsyncError/AsyncError.html' + 'https://api.dart.dev/stable/latest/dart-async/AsyncError/AsyncError.html' }, { 'path': 'dart-async/AsyncError/defaultStackTrace.html', 'url': - 'https://api.dart.dev/x/dart-async/AsyncError/defaultStackTrace.html' + 'https://api.dart.dev/stable/latest/dart-async/AsyncError/defaultStackTrace.html' }, ] }, @@ -124,13 +125,14 @@ void main() { 'sdk': 'dart', 'library': 'dart:async', 'description': 'async description', - 'url': 'https://api.dart.dev/x/dart-async/dart-async-library.html', + 'url': + 'https://api.dart.dev/stable/latest/dart-async/dart-async-library.html', 'score': closeTo(0.28, 0.01), 'apiPages': [ { 'path': 'dart-async/AsyncError/defaultStackTrace.html', 'url': - 'https://api.dart.dev/x/dart-async/AsyncError/defaultStackTrace.html' + 'https://api.dart.dev/stable/latest/dart-async/AsyncError/defaultStackTrace.html' }, ] }, @@ -147,13 +149,14 @@ void main() { 'sdk': 'dart', 'library': 'dart:async', 'description': 'async description', - 'url': 'https://api.dart.dev/x/dart-async/dart-async-library.html', + 'url': + 'https://api.dart.dev/stable/latest/dart-async/dart-async-library.html', 'score': closeTo(0.98, 0.01), 'apiPages': [ { 'path': 'dart-async/AsyncError/defaultStackTrace.html', 'url': - 'https://api.dart.dev/x/dart-async/AsyncError/defaultStackTrace.html' + 'https://api.dart.dev/stable/latest/dart-async/AsyncError/defaultStackTrace.html' }, ] },