Skip to content

Isolate-based access to the SDK index (but not used yet). #8793

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
May 28, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 13 additions & 1 deletion app/lib/search/sdk_mem_index.dart
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

import 'dart:async';
import 'dart:math';

import 'package:gcloud/service_scope.dart' as ss;
Expand Down Expand Up @@ -66,8 +67,18 @@ Future<SdkMemIndex?> createSdkMemIndex() async {
}
}

/// Defines the general interface for the SDK index.
// ignore: one_member_abstracts
abstract class SdkIndex {
FutureOr<List<SdkLibraryHit>> search(
String query, {
int? limit,
bool skipFlutter = false,
});
}

/// In-memory index for SDK library search queries.
class SdkMemIndex {
class SdkMemIndex implements SdkIndex {
final _libraries = <String, _Library>{};
final Map<String, double> _apiPageDirWeights;

Expand Down Expand Up @@ -126,6 +137,7 @@ class SdkMemIndex {
}
}

@override
List<SdkLibraryHit> search(
String query, {
int? limit,
Expand Down
30 changes: 30 additions & 0 deletions app/lib/service/entrypoint/_isolate.dart
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ import 'dart:isolate';

import 'package:collection/collection.dart';
import 'package:logging/logging.dart';
import 'package:pub_dev/service/entrypoint/tools.dart';
import 'package:pub_dev/shared/monitoring.dart';
import 'package:stack_trace/stack_trace.dart';

import '../services.dart';
Expand Down Expand Up @@ -325,3 +327,31 @@ Future<void> _wrapper(List args) async {
timer.cancel();
}
}

/// Run [fn] inside the isolate, using message passing from the control isolate.
Future<void> runIsolateFunctions({
required Object? message,
required Logger logger,
required Future<ReplyMessage> Function(Object payload) fn,
}) async {
final requestReceivePort = ReceivePort();
final entryMessage = Message.fromObject(message) as EntryMessage;

final subs = requestReceivePort.listen((e) async {
try {
final msg = Message.fromObject(e) as RequestMessage;
final reply = await fn(msg.payload);
msg.replyPort.send(reply.encodeAsJson());
} catch (e, st) {
logger.pubNoticeShout(
'isolate-message-error', 'Error processing message: $e', e, st);
}
});
entryMessage.protocolSendPort.send(
ReadyMessage(requestSendPort: requestReceivePort.sendPort).encodeAsJson(),
);

await waitForProcessSignalTermination();
requestReceivePort.close();
await subs.cancel();
}
81 changes: 81 additions & 0 deletions app/lib/service/entrypoint/sdk_isolate_index.dart
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

import 'dart:async';

import 'package:gcloud/service_scope.dart';
import 'package:logging/logging.dart';
import 'package:pub_dev/service/entrypoint/logging.dart';
import 'package:pub_dev/service/services.dart';
import 'package:pub_dev/shared/env_config.dart';
import 'package:pub_dev/shared/logging.dart';

import '../../../service/entrypoint/_isolate.dart';

import '../../search/sdk_mem_index.dart';
import '../../search/search_service.dart';

final _logger = Logger('sdk_search_index');

/// Entry point for the SDK search index isolate.
Future<void> main(List<String> args, var message) async {
final timer = Timer.periodic(Duration(milliseconds: 250), (_) {});

final ServicesWrapperFn servicesWrapperFn;
if (envConfig.isRunningInAppengine) {
servicesWrapperFn = withServices;
setupAppEngineLogging();
} else {
servicesWrapperFn = (fn) => withFakeServices(fn: fn);
setupDebugEnvBasedLogging();
}

await fork(() async {
await servicesWrapperFn(() async {
final sdkMemIndex = await createSdkMemIndex();
await runIsolateFunctions(
message: message,
logger: _logger,
fn: (payload) async {
final args = payload as List;
final rs = sdkMemIndex!.search(
args[0] as String,
limit: args[1] as int?,
skipFlutter: args[2] as bool,
);
return ReplyMessage.result(rs.map((e) => e.toJson()).toList());
});
});
});

timer.cancel();
}

/// Implementation of [SdkMemIndex] that uses [RequestMessage]s to send requests
/// across isolate boundaries. The instance should be registered inside the
/// `frontend` isolate, and it calls the `sdk-index` isolate as a delegate.
class SdkIsolateIndex implements SdkIndex {
final IsolateRunner _runner;
SdkIsolateIndex(this._runner);

@override
Future<List<SdkLibraryHit>> search(
String query, {
int? limit,
bool skipFlutter = false,
}) async {
try {
final rs = await _runner.sendRequest(
[query, limit, skipFlutter],
timeout: Duration(seconds: 2),
);
return (rs as List)
.map((v) => SdkLibraryHit.fromJson(v as Map<String, dynamic>))
.toList();
} catch (e, st) {
_logger.warning('Failed to search SDK index.', e, st);
return <SdkLibraryHit>[];
}
}
}
38 changes: 9 additions & 29 deletions app/lib/service/entrypoint/search_index.dart
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@

import 'dart:async';
import 'dart:convert';
import 'dart:isolate';

import 'package:gcloud/service_scope.dart';
import 'package:logging/logging.dart';
Expand Down Expand Up @@ -39,43 +38,24 @@ Future<void> main(List<String> args, var message) async {
registerSdkMemIndex(await createSdkMemIndex());
await indexUpdater.init();

final requestReceivePort = ReceivePort();
final entryMessage = Message.fromObject(message) as EntryMessage;

final subs = requestReceivePort.listen((e) async {
try {
final msg = Message.fromObject(e) as RequestMessage;
final payload = msg.payload;
await runIsolateFunctions(
message: message,
logger: _logger,
fn: (payload) async {
if (payload is String && payload == 'info') {
final info = await searchIndex.indexInfo();
msg.replyPort
.send(ReplyMessage.result(info.toJson()).encodeAsJson());
return;
return ReplyMessage.result(info.toJson());
} else if (payload is String) {
final q = ServiceSearchQuery.fromServiceUrl(Uri.parse(payload));
final rs = await searchIndex.search(q);
msg.replyPort.send(
ReplyMessage.result(json.encode(rs.toJson())).encodeAsJson());
return;
return ReplyMessage.result(json.encode(rs.toJson()));
} else {
_logger.pubNoticeShout(
'unknown-isolate-message', 'Unrecognized payload: $msg');
msg.replyPort.send(ReplyMessage.error('Unrecognized payload: $msg')
.encodeAsJson());
'unknown-isolate-message', 'Unrecognized payload: $payload');
return ReplyMessage.error('Unrecognized payload: $payload');
}
} catch (e, st) {
_logger.pubNoticeShout(
'isolate-message-error', 'Error processing message: $e', e, st);
}
});
entryMessage.protocolSendPort.send(
ReadyMessage(requestSendPort: requestReceivePort.sendPort)
.encodeAsJson(),
},
);

await Completer().future;
requestReceivePort.close();
await subs.cancel();
});
});

Expand Down
44 changes: 44 additions & 0 deletions app/test/service/entrypoint/sdk_isolate_index_test.dart
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

import 'package:logging/logging.dart';
import 'package:pub_dev/service/entrypoint/_isolate.dart';
import 'package:pub_dev/service/entrypoint/sdk_isolate_index.dart';
import 'package:test/test.dart';

final _logger = Logger('sdk_isolate_index_test');

void main() {
group('SDK index inside an isolate', () {
final indexRunner = IsolateRunner.uri(
kind: 'index',
logger: _logger,
spawnUri: Uri.parse(
'package:pub_dev/service/entrypoint/sdk_isolate_index.dart'),
);

tearDownAll(() async {
await indexRunner.close();
});

test('start and work with index', () async {
await indexRunner.start(1);

final sdkIndex = SdkIsolateIndex(indexRunner);

final rs = await sdkIndex.search('json');
expect(rs.map((e) => e.toJson()).toList(), [
{
'sdk': 'dart',
'library': 'dart:convert',
'description': isNotNull,
'url': 'https://api.dart.dev/stable/latest/dart-convert/',
'score': isNotNull,
'apiPages': isNotEmpty,
},
isA<Map>(), // second hit from `package:flutter_driver`
]);
}, timeout: Timeout(Duration(minutes: 5)));
});
}
3 changes: 3 additions & 0 deletions app/test/shared/timer_import_test.dart
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,9 @@ void main() {
// Uses timer to prevent GC compaction.
'lib/service/entrypoint/search_index.dart',

// Uses timer to prevent GC compaction.
'lib/service/entrypoint/sdk_isolate_index.dart',

// Uses timer to timeout GlobalLock claim acquisition.
'lib/tool/neat_task/pub_dev_tasks.dart',

Expand Down