From 3f316f657d461a907ac08e2e9e5c686e12a272f1 Mon Sep 17 00:00:00 2001 From: Istvan Soos Date: Thu, 15 May 2025 16:56:50 +0200 Subject: [PATCH] Refactor SDK index construction: removing mutable state. --- app/lib/search/dart_sdk_mem_index.dart | 13 +- app/lib/search/flutter_sdk_mem_index.dart | 16 +-- app/lib/search/sdk_mem_index.dart | 45 +++--- app/test/search/backend_test.dart | 7 +- .../search/dartdoc_index_parsing_test.dart | 3 +- app/test/search/result_combiner_test.dart | 85 ++++++----- app/test/search/sdk_mem_index_test.dart | 132 +++++++++--------- 7 files changed, 148 insertions(+), 153 deletions(-) diff --git a/app/lib/search/dart_sdk_mem_index.dart b/app/lib/search/dart_sdk_mem_index.dart index a16bf23856..690eaae4b9 100644 --- a/app/lib/search/dart_sdk_mem_index.dart +++ b/app/lib/search/dart_sdk_mem_index.dart @@ -4,7 +4,6 @@ import 'package:gcloud/service_scope.dart' as ss; import 'package:logging/logging.dart'; -import 'package:meta/meta.dart'; import 'backend.dart'; import 'sdk_mem_index.dart'; @@ -14,7 +13,6 @@ final _logger = Logger('search.dart_sdk_mem_index'); /// Results from these libraries are ranked with lower score and /// will be displayed only if the query has the library name, or /// there are not other results that could match the query. -@visibleForTesting const dartSdkLibraryWeights = { 'dart:html': 0.7, }; @@ -37,14 +35,9 @@ SdkMemIndex? get dartSdkMemIndex => /// was an error parsing the file or building the index. Future createDartSdkMemIndex() async { try { - final index = await SdkMemIndex.dart(); - final content = await loadOrFetchSdkIndexJsonAsString(index.indexJsonUri); - await index.addDartdocIndex(DartdocIndex.parseJsonText(content)); - index.updateWeights( - libraryWeights: dartSdkLibraryWeights, - apiPageDirWeights: {}, - ); - return index; + final content = + await loadOrFetchSdkIndexJsonAsString(SdkMemIndex.dartSdkIndexJsonUri); + return SdkMemIndex.dart(index: DartdocIndex.parseJsonText(content)); } catch (e, st) { _logger.warning('Unable to load Dart SDK index.', e, st); return null; diff --git a/app/lib/search/flutter_sdk_mem_index.dart b/app/lib/search/flutter_sdk_mem_index.dart index 4b4c2174ae..1089938e44 100644 --- a/app/lib/search/flutter_sdk_mem_index.dart +++ b/app/lib/search/flutter_sdk_mem_index.dart @@ -12,7 +12,7 @@ import 'sdk_mem_index.dart'; /// regular packages. The selected libraries are unique to the index.json. /// /// TODO: try to find a way to derive this list automatically. -const _allowedLibraries = { +const flutterSdkAllowedLibraries = { 'dart:ui', 'animation', 'cupertino', @@ -32,7 +32,7 @@ const _allowedLibraries = { 'flutter_web_plugins', }; -const _flutterApiPageDirWeights = { +const flutterApiPageDirWeights = { 'cupertino/CupertinoIcons': 0.25, 'material/Icons': 0.25, }; @@ -54,14 +54,10 @@ SdkMemIndex? get flutterSdkMemIndex => /// api.flutter.dev and returns search results based on [SdkMemIndex]. Future createFlutterSdkMemIndex() async { try { - final index = SdkMemIndex.flutter(); - final content = await loadOrFetchSdkIndexJsonAsString(index.indexJsonUri); - await index.addDartdocIndex(DartdocIndex.parseJsonText(content), - allowedLibraries: _allowedLibraries); - index.updateWeights( - libraryWeights: {}, - apiPageDirWeights: _flutterApiPageDirWeights, - ); + final content = await loadOrFetchSdkIndexJsonAsString( + SdkMemIndex.flutterSdkIndexJsonUri); + final index = + SdkMemIndex.flutter(index: DartdocIndex.parseJsonText(content)); return index; } catch (e, st) { _logger.warning('Unable to load Flutter SDK index.', e, st); diff --git a/app/lib/search/sdk_mem_index.dart b/app/lib/search/sdk_mem_index.dart index b865313af9..baa3f0cf09 100644 --- a/app/lib/search/sdk_mem_index.dart +++ b/app/lib/search/sdk_mem_index.dart @@ -8,6 +8,8 @@ import 'package:meta/meta.dart'; // ignore: implementation_imports import 'package:pana/src/dartdoc/dartdoc_index.dart'; import 'package:path/path.dart' as p; +import 'package:pub_dev/search/dart_sdk_mem_index.dart'; +import 'package:pub_dev/search/flutter_sdk_mem_index.dart'; import '../shared/versions.dart'; import 'search_service.dart'; @@ -23,39 +25,55 @@ class SdkMemIndex { final _tokensPerLibrary = >{}; final _baseUriPerLibrary = {}; final _descriptionPerLibrary = {}; - final _libraryWeights = {}; - final _apiPageDirWeights = {}; + final Map _libraryWeights; + final Map _apiPageDirWeights; SdkMemIndex({ required String sdk, required String? version, required Uri baseUri, + required DartdocIndex index, + Set? allowedLibraries, + Map? libraryWeights, + Map? apiPageDirWeights, }) : _sdk = sdk, _version = version, - _baseUri = baseUri; + _baseUri = baseUri, + _libraryWeights = libraryWeights ?? const {}, + _apiPageDirWeights = apiPageDirWeights ?? const {} { + _addDartdocIndex(index, allowedLibraries); + } - static Future dart() async { + static SdkMemIndex dart({required DartdocIndex index}) { return SdkMemIndex( sdk: 'dart', version: runtimeSdkVersion, baseUri: Uri.parse('https://api.dart.dev/stable/latest/'), + index: index, + libraryWeights: dartSdkLibraryWeights, ); } - factory SdkMemIndex.flutter() { + factory SdkMemIndex.flutter({required DartdocIndex index}) { return SdkMemIndex( sdk: 'flutter', version: null, baseUri: Uri.parse('https://api.flutter.dev/flutter/'), + index: index, + allowedLibraries: flutterSdkAllowedLibraries, + apiPageDirWeights: flutterApiPageDirWeights, ); } - late final indexJsonUri = _baseUri.resolve('index.json'); + static final dartSdkIndexJsonUri = + Uri.parse('https://api.dart.dev/stable/latest/index.json'); + static final flutterSdkIndexJsonUri = + Uri.parse('https://api.flutter.dev/flutter/index.json'); - Future addDartdocIndex( - DartdocIndex index, { + void _addDartdocIndex( + DartdocIndex index, Set? allowedLibraries, - }) async { + ) { final textsPerLibrary = >{}; for (final f in index.entries) { final library = f.qualifiedName?.split('.').first; @@ -86,15 +104,6 @@ class SdkMemIndex { } } - /// Updates the non-default weight for libraries. - void updateWeights({ - required Map libraryWeights, - required Map apiPageDirWeights, - }) { - _libraryWeights.addAll(libraryWeights); - _apiPageDirWeights.addAll(apiPageDirWeights); - } - List search( String query, { int? limit, diff --git a/app/test/search/backend_test.dart b/app/test/search/backend_test.dart index eee544da8a..84b5c2851a 100644 --- a/app/test/search/backend_test.dart +++ b/app/test/search/backend_test.dart @@ -13,9 +13,10 @@ import '../shared/test_services.dart'; void main() { group('search backend', () { testWithProfile('fetch SDK library description', fn: () async { - final index = await SdkMemIndex.dart(); - final content = await loadOrFetchSdkIndexJsonAsString(index.indexJsonUri); - await index.addDartdocIndex(DartdocIndex.parseJsonText(content)); + final content = await loadOrFetchSdkIndexJsonAsString( + SdkMemIndex.dartSdkIndexJsonUri); + final index = + SdkMemIndex.dart(index: DartdocIndex.parseJsonText(content)); expect( index.getLibraryDescription('dart:async'), 'Support for asynchronous programming, with classes such as Future and Stream.', diff --git a/app/test/search/dartdoc_index_parsing_test.dart b/app/test/search/dartdoc_index_parsing_test.dart index d5d9bb2f25..1ecbb2db41 100644 --- a/app/test/search/dartdoc_index_parsing_test.dart +++ b/app/test/search/dartdoc_index_parsing_test.dart @@ -66,8 +66,7 @@ void main() { expect(parserWithoutFirstEntry, originalWithoutFirstEntry); // parsing into SDK index - final sdkMemIndex = SdkMemIndex.flutter(); - await sdkMemIndex.addDartdocIndex(index); + final sdkMemIndex = SdkMemIndex.flutter(index: index); final rs = sdkMemIndex.search('StatelessWidget'); expect(json.decode(json.encode(rs)), [ { diff --git a/app/test/search/result_combiner_test.dart b/app/test/search/result_combiner_test.dart index 31fba08932..0b95d5995a 100644 --- a/app/test/search/result_combiner_test.dart +++ b/app/test/search/result_combiner_test.dart @@ -31,49 +31,48 @@ void main() { final combiner = SearchResultCombiner( primaryIndex: primaryIndex, dartSdkMemIndex: SdkMemIndex( - sdk: 'dart', - version: runtimeSdkVersion, - baseUri: Uri.parse('https://api.dart.dev/stable/$runtimeSdkVersion/')) - ..addDartdocIndex( - DartdocIndex.fromJsonList([ - { - 'name': 'dart:core', - 'qualifiedName': 'dart:core', - 'href': 'dart-core/dart-core-library.html', - 'kind': 8, - 'overriddenDepth': 0, - 'packageName': 'Dart' - }, - { - 'name': 'String', - 'qualifiedName': 'dart:core.String', - 'href': 'dart-core/String-class.html', - 'kind': 3, - 'overriddenDepth': 0, - 'packageName': 'Dart', - 'enclosedBy': {'name': 'dart:core', 'kind': 8} - }, - { - 'name': 'substring', - 'qualifiedName': 'dart:core.String.substring', - 'href': 'dart-core/String/substring.html', - 'kind': 9, - 'overriddenDepth': 0, - 'packageName': 'Dart', - 'enclosedBy': {'name': 'String', 'kind': 3} - }, - { - // fake method for checking the package name matches - 'name': 'stringutils', - 'qualifiedName': 'dart:core.String.stringutils', - 'href': 'dart-core/String/stringutils.html', - 'kind': 9, - 'overriddenDepth': 0, - 'packageName': 'Dart', - 'enclosedBy': {'name': 'String', 'kind': 3} - }, - ]), - ), + sdk: 'dart', + version: runtimeSdkVersion, + baseUri: Uri.parse('https://api.dart.dev/stable/$runtimeSdkVersion/'), + index: DartdocIndex.fromJsonList([ + { + 'name': 'dart:core', + 'qualifiedName': 'dart:core', + 'href': 'dart-core/dart-core-library.html', + 'kind': 8, + 'overriddenDepth': 0, + 'packageName': 'Dart' + }, + { + 'name': 'String', + 'qualifiedName': 'dart:core.String', + 'href': 'dart-core/String-class.html', + 'kind': 3, + 'overriddenDepth': 0, + 'packageName': 'Dart', + 'enclosedBy': {'name': 'dart:core', 'kind': 8} + }, + { + 'name': 'substring', + 'qualifiedName': 'dart:core.String.substring', + 'href': 'dart-core/String/substring.html', + 'kind': 9, + 'overriddenDepth': 0, + 'packageName': 'Dart', + 'enclosedBy': {'name': 'String', 'kind': 3} + }, + { + // fake method for checking the package name matches + 'name': 'stringutils', + 'qualifiedName': 'dart:core.String.stringutils', + 'href': 'dart-core/String/stringutils.html', + 'kind': 9, + 'overriddenDepth': 0, + 'packageName': 'Dart', + 'enclosedBy': {'name': 'String', 'kind': 3} + }, + ]), + ), flutterSdkMemIndex: null, ); diff --git a/app/test/search/sdk_mem_index_test.dart b/app/test/search/sdk_mem_index_test.dart index 8ea321ce92..2c4f546af2 100644 --- a/app/test/search/sdk_mem_index_test.dart +++ b/app/test/search/sdk_mem_index_test.dart @@ -17,78 +17,76 @@ void main() { sdk: 'dart', version: '', baseUri: Uri.parse('https://api.dart.dev/x/'), - ); - index.updateWeights( libraryWeights: dartSdkLibraryWeights, apiPageDirWeights: { 'dart:html/FakeIcons': 0.7, }, + index: DartdocIndex.fromJsonList([ + { + 'name': 'dart:async', + 'qualifiedName': 'dart:async', + 'href': 'dart-async/dart-async-library.html', + 'kind': 8, + 'overriddenDepth': 0, + 'packageName': 'Dart', + 'desc': 'async description', + }, + { + 'name': 'AsyncError', + 'qualifiedName': 'dart:async.AsyncError', + 'href': 'dart-async/AsyncError-class.html', + 'kind': 3, + 'overriddenDepth': 0, + 'packageName': 'Dart', + 'enclosedBy': {'name': 'dart:async', 'kind': 8}, + }, + { + 'name': 'AsyncError', + 'qualifiedName': 'dart:async.AsyncError.AsyncError', + 'href': 'dart-async/AsyncError/AsyncError.html', + 'kind': 2, + 'overriddenDepth': 0, + 'packageName': 'Dart', + 'enclosedBy': {'name': 'AsyncError', 'kind': 3}, + }, + { + 'name': 'defaultStackTrace', + 'qualifiedName': 'dart:async.AsyncError.defaultStackTrace', + 'href': 'dart-async/AsyncError/defaultStackTrace.html', + 'kind': 9, + 'overriddenDepth': 0, + 'packageName': 'Dart', + 'enclosedBy': {'name': 'AsyncError', 'kind': 3}, + }, + { + 'name': 'dart:html', + 'qualifiedName': 'dart:html', + 'href': 'dart-html', + 'kind': 8, + 'overriddenDepth': 0, + 'packageName': 'HTML', + 'desc': 'html description', + }, + { + 'name': 'Window', + 'qualifiedName': 'dart:html.Window', + 'href': 'dart-html/Window-class.html', + 'kind': 3, + 'overriddenDepth': 0, + 'packageName': 'Dart', + 'enclosedBy': {'name': 'dart:html', 'kind': 8}, + }, + { + 'name': 'flame', + 'qualifiedName': 'dart:html.FakeIcons.flame', + 'href': 'dart-html/FakeIcons/flame-constant.html', + 'kind': 1, + 'overriddenDepth': 0, + 'packageName': 'Dart', + 'enclosedBy': {'name': 'dart:html.FakeIcons', 'kind': 3}, + }, + ]), ); - await index.addDartdocIndex(DartdocIndex.fromJsonList([ - { - 'name': 'dart:async', - 'qualifiedName': 'dart:async', - 'href': 'dart-async/dart-async-library.html', - 'kind': 8, - 'overriddenDepth': 0, - 'packageName': 'Dart', - 'desc': 'async description', - }, - { - 'name': 'AsyncError', - 'qualifiedName': 'dart:async.AsyncError', - 'href': 'dart-async/AsyncError-class.html', - 'kind': 3, - 'overriddenDepth': 0, - 'packageName': 'Dart', - 'enclosedBy': {'name': 'dart:async', 'kind': 8}, - }, - { - 'name': 'AsyncError', - 'qualifiedName': 'dart:async.AsyncError.AsyncError', - 'href': 'dart-async/AsyncError/AsyncError.html', - 'kind': 2, - 'overriddenDepth': 0, - 'packageName': 'Dart', - 'enclosedBy': {'name': 'AsyncError', 'kind': 3}, - }, - { - 'name': 'defaultStackTrace', - 'qualifiedName': 'dart:async.AsyncError.defaultStackTrace', - 'href': 'dart-async/AsyncError/defaultStackTrace.html', - 'kind': 9, - 'overriddenDepth': 0, - 'packageName': 'Dart', - 'enclosedBy': {'name': 'AsyncError', 'kind': 3}, - }, - { - 'name': 'dart:html', - 'qualifiedName': 'dart:html', - 'href': 'dart-html', - 'kind': 8, - 'overriddenDepth': 0, - 'packageName': 'HTML', - 'desc': 'html description', - }, - { - 'name': 'Window', - 'qualifiedName': 'dart:html.Window', - 'href': 'dart-html/Window-class.html', - 'kind': 3, - 'overriddenDepth': 0, - 'packageName': 'Dart', - 'enclosedBy': {'name': 'dart:html', 'kind': 8}, - }, - { - 'name': 'flame', - 'qualifiedName': 'dart:html.FakeIcons.flame', - 'href': 'dart-html/FakeIcons/flame-constant.html', - 'kind': 1, - 'overriddenDepth': 0, - 'packageName': 'Dart', - 'enclosedBy': {'name': 'dart:html.FakeIcons', 'kind': 3}, - }, - ])); }); test('AsyncError', () async {