diff --git a/app/lib/search/backend.dart b/app/lib/search/backend.dart index 7688f68406..b9fc34a7e3 100644 --- a/app/lib/search/backend.dart +++ b/app/lib/search/backend.dart @@ -87,7 +87,6 @@ void registerSearchIndex(SearchIndex index) => class SearchBackend { final DatastoreDB _db; final VersionedJsonStorage _snapshotStorage; - final _http = httpRetryClient(); SearchBackend(this._db, Bucket snapshotBucket) : _snapshotStorage = VersionedJsonStorage(snapshotBucket, 'snapshot/'); @@ -428,42 +427,6 @@ class SearchBackend { } } - /// Downloads the remote SDK content from [uri] and creates a cached file in the - /// `.dart_tool/pub-search-data/` directory. - /// - /// When a local file in `app/.dart_tool/pub-search-data/` exists, - /// its content will be loaded instead. URL reduction replaces slashes and other - /// non-characters with a single dash `-`, like: - /// - `https-api.dart.dev-stable-latest-index.json` - Future loadOrFetchSdkIndexJsonAsString( - Uri uri, { - @visibleForTesting Duration? ttl, - }) async { - final fileName = uri.toString().replaceAll(RegExp(r'[^a-z0-9\.]+'), '-'); - final file = File(p.join('.dart_tool', 'pub-search-data', fileName)); - if (await file.exists()) { - var canUseCached = true; - if (ttl != null) { - final age = clock.now().difference(await file.lastModified()); - if (age > ttl) { - canUseCached = false; - } - } - if (canUseCached) { - return await file.readAsString(); - } - } - - final rs = await _http.get(uri); - if (rs.statusCode != 200) { - throw Exception('Unexpected status code for $uri: ${rs.statusCode}'); - } - final content = rs.body; - await file.parent.create(recursive: true); - await file.writeAsString(content); - return content; - } - Future?> fetchSnapshotDocuments() async { try { final map = await _snapshotStorage.getContentAsJsonMap(); @@ -519,7 +482,6 @@ class SearchBackend { Future close() async { _snapshotStorage.close(); - _http.close(); } } @@ -615,6 +577,38 @@ List apiDocPagesFromPubData(PubDartdocData pubData) { return results; } +/// Downloads the remote SDK content from [uri] and creates a cached file in the +/// `.dart_tool/pub-search-data/` directory. +/// +/// When a local file in `app/.dart_tool/pub-search-data/` exists, +/// its content will be loaded instead. URL reduction replaces slashes and other +/// non-characters with a single dash `-`, like: +/// - `https-api.dart.dev-stable-latest-index.json` +Future loadOrFetchSdkIndexJsonAsString( + Uri uri, { + @visibleForTesting Duration? ttl, +}) async { + final fileName = uri.toString().replaceAll(RegExp(r'[^a-z0-9\.]+'), '-'); + final file = File(p.join('.dart_tool', 'pub-search-data', fileName)); + if (await file.exists()) { + var canUseCached = true; + if (ttl != null) { + final age = clock.now().difference(await file.lastModified()); + if (age > ttl) { + canUseCached = false; + } + } + if (canUseCached) { + return await file.readAsString(); + } + } + + final content = await httpGetWithRetry(uri, responseFn: (rs) => rs.body); + await file.parent.create(recursive: true); + await file.writeAsString(content); + return content; +} + class _CombinedSearchIndex implements SearchIndex { const _CombinedSearchIndex(); diff --git a/app/lib/search/dart_sdk_mem_index.dart b/app/lib/search/dart_sdk_mem_index.dart index 22c291c551..a16bf23856 100644 --- a/app/lib/search/dart_sdk_mem_index.dart +++ b/app/lib/search/dart_sdk_mem_index.dart @@ -38,8 +38,7 @@ SdkMemIndex? get dartSdkMemIndex => Future createDartSdkMemIndex() async { try { final index = await SdkMemIndex.dart(); - final content = - await searchBackend.loadOrFetchSdkIndexJsonAsString(index.indexJsonUri); + final content = await loadOrFetchSdkIndexJsonAsString(index.indexJsonUri); await index.addDartdocIndex(DartdocIndex.parseJsonText(content)); index.updateWeights( libraryWeights: dartSdkLibraryWeights, diff --git a/app/lib/search/flutter_sdk_mem_index.dart b/app/lib/search/flutter_sdk_mem_index.dart index 1aebf08423..4b4c2174ae 100644 --- a/app/lib/search/flutter_sdk_mem_index.dart +++ b/app/lib/search/flutter_sdk_mem_index.dart @@ -55,8 +55,7 @@ SdkMemIndex? get flutterSdkMemIndex => Future createFlutterSdkMemIndex() async { try { final index = SdkMemIndex.flutter(); - final content = - await searchBackend.loadOrFetchSdkIndexJsonAsString(index.indexJsonUri); + final content = await loadOrFetchSdkIndexJsonAsString(index.indexJsonUri); await index.addDartdocIndex(DartdocIndex.parseJsonText(content), allowedLibraries: _allowedLibraries); index.updateWeights( diff --git a/app/test/search/backend_test.dart b/app/test/search/backend_test.dart index c6a7a2a476..eee544da8a 100644 --- a/app/test/search/backend_test.dart +++ b/app/test/search/backend_test.dart @@ -14,8 +14,7 @@ void main() { group('search backend', () { testWithProfile('fetch SDK library description', fn: () async { final index = await SdkMemIndex.dart(); - final content = await searchBackend - .loadOrFetchSdkIndexJsonAsString(index.indexJsonUri); + final content = await loadOrFetchSdkIndexJsonAsString(index.indexJsonUri); await index.addDartdocIndex(DartdocIndex.parseJsonText(content)); expect( index.getLibraryDescription('dart:async'), diff --git a/app/test/search/dartdoc_index_parsing_test.dart b/app/test/search/dartdoc_index_parsing_test.dart index 76d79c4368..d5d9bb2f25 100644 --- a/app/test/search/dartdoc_index_parsing_test.dart +++ b/app/test/search/dartdoc_index_parsing_test.dart @@ -13,7 +13,7 @@ import '../shared/test_services.dart'; void main() { group('dartdoc index.json parsing', () { testWithProfile('parse Dart SDK index.json', fn: () async { - final textContent = await searchBackend.loadOrFetchSdkIndexJsonAsString( + final textContent = await loadOrFetchSdkIndexJsonAsString( Uri.parse('https://api.dart.dev/stable/latest/index.json'), ttl: Duration(days: 1), ); @@ -35,7 +35,7 @@ void main() { }); testWithProfile('parse Flutter SDK index.json', fn: () async { - final textContent = await searchBackend.loadOrFetchSdkIndexJsonAsString( + final textContent = await loadOrFetchSdkIndexJsonAsString( Uri.parse('https://api.flutter.dev/flutter/index.json'), ttl: Duration(days: 1), ); diff --git a/pkg/_pub_shared/lib/utils/http.dart b/pkg/_pub_shared/lib/utils/http.dart index fa11e1d09b..5876eb468b 100644 --- a/pkg/_pub_shared/lib/utils/http.dart +++ b/pkg/_pub_shared/lib/utils/http.dart @@ -2,10 +2,12 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +import 'dart:async'; import 'dart:io'; import 'package:http/http.dart' as http; import 'package:http/retry.dart'; +import 'package:retry/retry.dart'; final _transientStatusCodes = { // See: https://cloud.google.com/storage/docs/xml-api/reference-status @@ -35,3 +37,45 @@ http.Client httpRetryClient({ whenError: (e, st) => lenient || e is SocketException, ); } + +/// Creates a HTTP client and executes a GET request to the specified [uri], +/// making sure that the HTTP resources are freed after the [responseFn] +/// callback finishes. +/// The HTTP GET and the [responseFn] callback is retried on the transient +/// network errors. +Future httpGetWithRetry( + Uri uri, { + required FutureOr Function(http.Response response) responseFn, + int maxAttempts = 3, +}) async { + return await retry( + () async { + final client = http.Client(); + try { + final rs = await client.get(uri); + if (rs.statusCode == 200) { + return responseFn(rs); + } + throw http.ClientException( + 'Unexpected status code for $uri: ${rs.statusCode}.'); + } finally { + client.close(); + } + }, + maxAttempts: maxAttempts, + retryIf: _retryIf, + ); +} + +bool _retryIf(Exception e) { + if (e is TimeoutException) { + return true; // Timeouts we can retry + } + if (e is IOException) { + return true; // I/O issues are worth retrying + } + if (e is http.ClientException) { + return true; // HTTP issues are worth retrying + } + return false; +} diff --git a/pkg/_pub_shared/pubspec.yaml b/pkg/_pub_shared/pubspec.yaml index 6ef30e56fd..f5ba5c0010 100644 --- a/pkg/_pub_shared/pubspec.yaml +++ b/pkg/_pub_shared/pubspec.yaml @@ -13,6 +13,7 @@ dependencies: logging: ^1.2.0 meta: ^1.3.0 pub_semver: ^2.0.0 + retry: ^3.1.2 sanitize_html: ^2.1.0 api_builder: