From d86b3703acd0aac86f760ca1b9d4e810d51fdb9f Mon Sep 17 00:00:00 2001 From: Istvan Soos Date: Tue, 27 May 2025 18:04:04 +0200 Subject: [PATCH] Isolate-based access to the SDK index (but not used yet). --- app/lib/search/sdk_mem_index.dart | 14 +++- app/lib/service/entrypoint/_isolate.dart | 30 +++++++ .../service/entrypoint/sdk_isolate_index.dart | 81 +++++++++++++++++++ app/lib/service/entrypoint/search_index.dart | 38 +++------ .../entrypoint/sdk_isolate_index_test.dart | 44 ++++++++++ app/test/shared/timer_import_test.dart | 3 + 6 files changed, 180 insertions(+), 30 deletions(-) create mode 100644 app/lib/service/entrypoint/sdk_isolate_index.dart create mode 100644 app/test/service/entrypoint/sdk_isolate_index_test.dart diff --git a/app/lib/search/sdk_mem_index.dart b/app/lib/search/sdk_mem_index.dart index 397ff5d513..c974dad37d 100644 --- a/app/lib/search/sdk_mem_index.dart +++ b/app/lib/search/sdk_mem_index.dart @@ -2,6 +2,7 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +import 'dart:async'; import 'dart:math'; import 'package:gcloud/service_scope.dart' as ss; @@ -66,8 +67,18 @@ Future createSdkMemIndex() async { } } +/// Defines the general interface for the SDK index. +// ignore: one_member_abstracts +abstract class SdkIndex { + FutureOr> search( + String query, { + int? limit, + bool skipFlutter = false, + }); +} + /// In-memory index for SDK library search queries. -class SdkMemIndex { +class SdkMemIndex implements SdkIndex { final _libraries = {}; final Map _apiPageDirWeights; @@ -126,6 +137,7 @@ class SdkMemIndex { } } + @override List search( String query, { int? limit, diff --git a/app/lib/service/entrypoint/_isolate.dart b/app/lib/service/entrypoint/_isolate.dart index 98806cb4df..f615c06a75 100644 --- a/app/lib/service/entrypoint/_isolate.dart +++ b/app/lib/service/entrypoint/_isolate.dart @@ -8,6 +8,8 @@ import 'dart:isolate'; import 'package:collection/collection.dart'; import 'package:logging/logging.dart'; +import 'package:pub_dev/service/entrypoint/tools.dart'; +import 'package:pub_dev/shared/monitoring.dart'; import 'package:stack_trace/stack_trace.dart'; import '../services.dart'; @@ -325,3 +327,31 @@ Future _wrapper(List args) async { timer.cancel(); } } + +/// Run [fn] inside the isolate, using message passing from the control isolate. +Future runIsolateFunctions({ + required Object? message, + required Logger logger, + required Future Function(Object payload) fn, +}) async { + final requestReceivePort = ReceivePort(); + final entryMessage = Message.fromObject(message) as EntryMessage; + + final subs = requestReceivePort.listen((e) async { + try { + final msg = Message.fromObject(e) as RequestMessage; + final reply = await fn(msg.payload); + msg.replyPort.send(reply.encodeAsJson()); + } catch (e, st) { + logger.pubNoticeShout( + 'isolate-message-error', 'Error processing message: $e', e, st); + } + }); + entryMessage.protocolSendPort.send( + ReadyMessage(requestSendPort: requestReceivePort.sendPort).encodeAsJson(), + ); + + await waitForProcessSignalTermination(); + requestReceivePort.close(); + await subs.cancel(); +} diff --git a/app/lib/service/entrypoint/sdk_isolate_index.dart b/app/lib/service/entrypoint/sdk_isolate_index.dart new file mode 100644 index 0000000000..b848a0a43b --- /dev/null +++ b/app/lib/service/entrypoint/sdk_isolate_index.dart @@ -0,0 +1,81 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:async'; + +import 'package:gcloud/service_scope.dart'; +import 'package:logging/logging.dart'; +import 'package:pub_dev/service/entrypoint/logging.dart'; +import 'package:pub_dev/service/services.dart'; +import 'package:pub_dev/shared/env_config.dart'; +import 'package:pub_dev/shared/logging.dart'; + +import '../../../service/entrypoint/_isolate.dart'; + +import '../../search/sdk_mem_index.dart'; +import '../../search/search_service.dart'; + +final _logger = Logger('sdk_search_index'); + +/// Entry point for the SDK search index isolate. +Future main(List args, var message) async { + final timer = Timer.periodic(Duration(milliseconds: 250), (_) {}); + + final ServicesWrapperFn servicesWrapperFn; + if (envConfig.isRunningInAppengine) { + servicesWrapperFn = withServices; + setupAppEngineLogging(); + } else { + servicesWrapperFn = (fn) => withFakeServices(fn: fn); + setupDebugEnvBasedLogging(); + } + + await fork(() async { + await servicesWrapperFn(() async { + final sdkMemIndex = await createSdkMemIndex(); + await runIsolateFunctions( + message: message, + logger: _logger, + fn: (payload) async { + final args = payload as List; + final rs = sdkMemIndex!.search( + args[0] as String, + limit: args[1] as int?, + skipFlutter: args[2] as bool, + ); + return ReplyMessage.result(rs.map((e) => e.toJson()).toList()); + }); + }); + }); + + timer.cancel(); +} + +/// Implementation of [SdkMemIndex] that uses [RequestMessage]s to send requests +/// across isolate boundaries. The instance should be registered inside the +/// `frontend` isolate, and it calls the `sdk-index` isolate as a delegate. +class SdkIsolateIndex implements SdkIndex { + final IsolateRunner _runner; + SdkIsolateIndex(this._runner); + + @override + Future> search( + String query, { + int? limit, + bool skipFlutter = false, + }) async { + try { + final rs = await _runner.sendRequest( + [query, limit, skipFlutter], + timeout: Duration(seconds: 2), + ); + return (rs as List) + .map((v) => SdkLibraryHit.fromJson(v as Map)) + .toList(); + } catch (e, st) { + _logger.warning('Failed to search SDK index.', e, st); + return []; + } + } +} diff --git a/app/lib/service/entrypoint/search_index.dart b/app/lib/service/entrypoint/search_index.dart index f6a3a6e6f8..8ed37b6094 100644 --- a/app/lib/service/entrypoint/search_index.dart +++ b/app/lib/service/entrypoint/search_index.dart @@ -4,7 +4,6 @@ import 'dart:async'; import 'dart:convert'; -import 'dart:isolate'; import 'package:gcloud/service_scope.dart'; import 'package:logging/logging.dart'; @@ -39,43 +38,24 @@ Future main(List args, var message) async { registerSdkMemIndex(await createSdkMemIndex()); await indexUpdater.init(); - final requestReceivePort = ReceivePort(); - final entryMessage = Message.fromObject(message) as EntryMessage; - - final subs = requestReceivePort.listen((e) async { - try { - final msg = Message.fromObject(e) as RequestMessage; - final payload = msg.payload; + await runIsolateFunctions( + message: message, + logger: _logger, + fn: (payload) async { if (payload is String && payload == 'info') { final info = await searchIndex.indexInfo(); - msg.replyPort - .send(ReplyMessage.result(info.toJson()).encodeAsJson()); - return; + return ReplyMessage.result(info.toJson()); } else if (payload is String) { final q = ServiceSearchQuery.fromServiceUrl(Uri.parse(payload)); final rs = await searchIndex.search(q); - msg.replyPort.send( - ReplyMessage.result(json.encode(rs.toJson())).encodeAsJson()); - return; + return ReplyMessage.result(json.encode(rs.toJson())); } else { _logger.pubNoticeShout( - 'unknown-isolate-message', 'Unrecognized payload: $msg'); - msg.replyPort.send(ReplyMessage.error('Unrecognized payload: $msg') - .encodeAsJson()); + 'unknown-isolate-message', 'Unrecognized payload: $payload'); + return ReplyMessage.error('Unrecognized payload: $payload'); } - } catch (e, st) { - _logger.pubNoticeShout( - 'isolate-message-error', 'Error processing message: $e', e, st); - } - }); - entryMessage.protocolSendPort.send( - ReadyMessage(requestSendPort: requestReceivePort.sendPort) - .encodeAsJson(), + }, ); - - await Completer().future; - requestReceivePort.close(); - await subs.cancel(); }); }); diff --git a/app/test/service/entrypoint/sdk_isolate_index_test.dart b/app/test/service/entrypoint/sdk_isolate_index_test.dart new file mode 100644 index 0000000000..7257e92659 --- /dev/null +++ b/app/test/service/entrypoint/sdk_isolate_index_test.dart @@ -0,0 +1,44 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'package:logging/logging.dart'; +import 'package:pub_dev/service/entrypoint/_isolate.dart'; +import 'package:pub_dev/service/entrypoint/sdk_isolate_index.dart'; +import 'package:test/test.dart'; + +final _logger = Logger('sdk_isolate_index_test'); + +void main() { + group('SDK index inside an isolate', () { + final indexRunner = IsolateRunner.uri( + kind: 'index', + logger: _logger, + spawnUri: Uri.parse( + 'package:pub_dev/service/entrypoint/sdk_isolate_index.dart'), + ); + + tearDownAll(() async { + await indexRunner.close(); + }); + + test('start and work with index', () async { + await indexRunner.start(1); + + final sdkIndex = SdkIsolateIndex(indexRunner); + + final rs = await sdkIndex.search('json'); + expect(rs.map((e) => e.toJson()).toList(), [ + { + 'sdk': 'dart', + 'library': 'dart:convert', + 'description': isNotNull, + 'url': 'https://api.dart.dev/stable/latest/dart-convert/', + 'score': isNotNull, + 'apiPages': isNotEmpty, + }, + isA(), // second hit from `package:flutter_driver` + ]); + }, timeout: Timeout(Duration(minutes: 5))); + }); +} diff --git a/app/test/shared/timer_import_test.dart b/app/test/shared/timer_import_test.dart index 8bfda78d1f..bd38c5fe88 100644 --- a/app/test/shared/timer_import_test.dart +++ b/app/test/shared/timer_import_test.dart @@ -44,6 +44,9 @@ void main() { // Uses timer to prevent GC compaction. 'lib/service/entrypoint/search_index.dart', + // Uses timer to prevent GC compaction. + 'lib/service/entrypoint/sdk_isolate_index.dart', + // Uses timer to timeout GlobalLock claim acquisition. 'lib/tool/neat_task/pub_dev_tasks.dart',