diff --git a/Dockerfile.app b/Dockerfile.app index 68d5b45739..7fd0b143cc 100644 --- a/Dockerfile.app +++ b/Dockerfile.app @@ -33,6 +33,7 @@ WORKDIR /project/app RUN dart /project/tool/pub_get_offline.dart /project/app RUN /project/tool/setup-webp.sh /usr/local/bin +RUN /project/tool/download-sdk-index-jsons.sh # Clear out any arguments the base images might have set CMD [] diff --git a/app/lib/search/backend.dart b/app/lib/search/backend.dart index 5d9f42ae7a..7688f68406 100644 --- a/app/lib/search/backend.dart +++ b/app/lib/search/backend.dart @@ -17,6 +17,7 @@ import 'package:logging/logging.dart'; import 'package:meta/meta.dart'; // ignore: implementation_imports import 'package:pana/src/dartdoc/pub_dartdoc_data.dart'; +import 'package:path/path.dart' as p; import 'package:pool/pool.dart'; import 'package:pub_dev/shared/monitoring.dart'; import 'package:retry/retry.dart'; @@ -427,17 +428,40 @@ class SearchBackend { } } - /// Downloads the remote SDK content relative to the base URI. - Future fetchSdkIndexContentAsString({ - required Uri baseUri, - required String relativePath, + /// Downloads the remote SDK content from [uri] and creates a cached file in the + /// `.dart_tool/pub-search-data/` directory. + /// + /// When a local file in `app/.dart_tool/pub-search-data/` exists, + /// its content will be loaded instead. URL reduction replaces slashes and other + /// non-characters with a single dash `-`, like: + /// - `https-api.dart.dev-stable-latest-index.json` + Future loadOrFetchSdkIndexJsonAsString( + Uri uri, { + @visibleForTesting Duration? ttl, }) async { - final uri = baseUri.resolve(relativePath); + final fileName = uri.toString().replaceAll(RegExp(r'[^a-z0-9\.]+'), '-'); + final file = File(p.join('.dart_tool', 'pub-search-data', fileName)); + if (await file.exists()) { + var canUseCached = true; + if (ttl != null) { + final age = clock.now().difference(await file.lastModified()); + if (age > ttl) { + canUseCached = false; + } + } + if (canUseCached) { + return await file.readAsString(); + } + } + final rs = await _http.get(uri); if (rs.statusCode != 200) { throw Exception('Unexpected status code for $uri: ${rs.statusCode}'); } - return rs.body; + final content = rs.body; + await file.parent.create(recursive: true); + await file.writeAsString(content); + return content; } Future?> fetchSnapshotDocuments() async { diff --git a/app/lib/search/dart_sdk_mem_index.dart b/app/lib/search/dart_sdk_mem_index.dart index e44c7940ac..22c291c551 100644 --- a/app/lib/search/dart_sdk_mem_index.dart +++ b/app/lib/search/dart_sdk_mem_index.dart @@ -38,10 +38,8 @@ SdkMemIndex? get dartSdkMemIndex => Future createDartSdkMemIndex() async { try { final index = await SdkMemIndex.dart(); - final content = await searchBackend.fetchSdkIndexContentAsString( - baseUri: index.baseUri, - relativePath: 'index.json', - ); + final content = + await searchBackend.loadOrFetchSdkIndexJsonAsString(index.indexJsonUri); await index.addDartdocIndex(DartdocIndex.parseJsonText(content)); index.updateWeights( libraryWeights: dartSdkLibraryWeights, diff --git a/app/lib/search/flutter_sdk_mem_index.dart b/app/lib/search/flutter_sdk_mem_index.dart index bb6e89958b..1aebf08423 100644 --- a/app/lib/search/flutter_sdk_mem_index.dart +++ b/app/lib/search/flutter_sdk_mem_index.dart @@ -55,10 +55,8 @@ SdkMemIndex? get flutterSdkMemIndex => Future createFlutterSdkMemIndex() async { try { final index = SdkMemIndex.flutter(); - final content = await searchBackend.fetchSdkIndexContentAsString( - baseUri: index.baseUri, - relativePath: 'index.json', - ); + final content = + await searchBackend.loadOrFetchSdkIndexJsonAsString(index.indexJsonUri); await index.addDartdocIndex(DartdocIndex.parseJsonText(content), allowedLibraries: _allowedLibraries); index.updateWeights( diff --git a/app/lib/search/sdk_mem_index.dart b/app/lib/search/sdk_mem_index.dart index 1c1332e6b2..b865313af9 100644 --- a/app/lib/search/sdk_mem_index.dart +++ b/app/lib/search/sdk_mem_index.dart @@ -4,7 +4,6 @@ import 'dart:math'; -import 'package:_pub_shared/utils/http.dart'; import 'package:meta/meta.dart'; // ignore: implementation_imports import 'package:pana/src/dartdoc/dartdoc_index.dart'; @@ -36,39 +35,13 @@ class SdkMemIndex { _baseUri = baseUri; static Future dart() async { - final versions = { - toolStableDartSdkVersion, - runtimeSdkVersion, - }; - final client = httpRetryClient(); - try { - for (final version in versions) { - final uri = _dartSdkBaseUri(version); - final rs = await client.head(uri); - if (rs.statusCode < 400) { - return SdkMemIndex(sdk: 'dart', version: version, baseUri: uri); - } - } - } finally { - client.close(); - } return SdkMemIndex( sdk: 'dart', version: runtimeSdkVersion, - baseUri: _dartSdkBaseUri(runtimeSdkVersion), + baseUri: Uri.parse('https://api.dart.dev/stable/latest/'), ); } - static Uri _dartSdkBaseUri(String version) { - var branch = 'stable'; - if (version.contains('beta')) { - branch = 'beta'; - } else if (version.contains('dev')) { - branch = 'dev'; - } - return Uri.parse('https://api.dart.dev/$branch/$version/'); - } - factory SdkMemIndex.flutter() { return SdkMemIndex( sdk: 'flutter', @@ -77,7 +50,7 @@ class SdkMemIndex { ); } - Uri get baseUri => _baseUri; + late final indexJsonUri = _baseUri.resolve('index.json'); Future addDartdocIndex( DartdocIndex index, { diff --git a/app/test/search/backend_test.dart b/app/test/search/backend_test.dart index 4152694ecc..c6a7a2a476 100644 --- a/app/test/search/backend_test.dart +++ b/app/test/search/backend_test.dart @@ -14,10 +14,8 @@ void main() { group('search backend', () { testWithProfile('fetch SDK library description', fn: () async { final index = await SdkMemIndex.dart(); - final content = await searchBackend.fetchSdkIndexContentAsString( - baseUri: index.baseUri, - relativePath: 'index.json', - ); + final content = await searchBackend + .loadOrFetchSdkIndexJsonAsString(index.indexJsonUri); await index.addDartdocIndex(DartdocIndex.parseJsonText(content)); expect( index.getLibraryDescription('dart:async'), diff --git a/app/test/search/dartdoc_index_parsing_test.dart b/app/test/search/dartdoc_index_parsing_test.dart index 93079ace00..76d79c4368 100644 --- a/app/test/search/dartdoc_index_parsing_test.dart +++ b/app/test/search/dartdoc_index_parsing_test.dart @@ -3,46 +3,21 @@ // BSD-style license that can be found in the LICENSE file. import 'dart:convert'; -import 'dart:io'; -import 'package:clock/clock.dart'; -import 'package:http/http.dart' as http; -import 'package:path/path.dart' as p; +import 'package:pub_dev/search/backend.dart'; import 'package:pub_dev/search/sdk_mem_index.dart'; -import 'package:pub_dev/shared/versions.dart'; -import 'package:retry/retry.dart'; import 'package:test/test.dart'; +import '../shared/test_services.dart'; + void main() { group('dartdoc index.json parsing', () { - /// Downloads [url and creates a cached file in the .dart_tool/pub-search-data directory. - /// - /// Reuses the same file up to a week. - Future getCachedFile(String name, String url) async { - final file = File(p.join('.dart_tool', 'pub-search-data', name)); - if (await file.exists()) { - final lastModified = await file.lastModified(); - final age = clock.now().difference(lastModified); - if (age.inDays < 7) { - return file; - } - } - await file.parent.create(recursive: true); - return retry(() async { - final rs = await http.get(Uri.parse(url)); - if (rs.statusCode != 200) { - throw Exception('Unexpected status code for $url: ${rs.statusCode}'); - } - await file.writeAsBytes(rs.bodyBytes); - return file; - }); - } - - test('parse Dart SDK index.json', () async { - final file = await getCachedFile('dart-sdk-$runtimeSdkVersion.json', - 'https://api.dart.dev/stable/$runtimeSdkVersion/index.json'); - final textContent = await file.readAsString(); - final index = DartdocIndex.parseJsonText(await file.readAsString()); + testWithProfile('parse Dart SDK index.json', fn: () async { + final textContent = await searchBackend.loadOrFetchSdkIndexJsonAsString( + Uri.parse('https://api.dart.dev/stable/latest/index.json'), + ttl: Duration(days: 1), + ); + final index = DartdocIndex.parseJsonText(textContent); expect(index.entries, hasLength(greaterThan(10000))); final libraries = @@ -59,11 +34,12 @@ void main() { expect(json.decode(index.toJsonText()), json.decode(textContent)); }); - test('parse Flutter SDK index.json', () async { - final file = await getCachedFile( - 'flutter-sdk.json', 'https://api.flutter.dev/flutter/index.json'); - final textContent = await file.readAsString(); - final index = DartdocIndex.parseJsonText(await file.readAsString()); + testWithProfile('parse Flutter SDK index.json', fn: () async { + final textContent = await searchBackend.loadOrFetchSdkIndexJsonAsString( + Uri.parse('https://api.flutter.dev/flutter/index.json'), + ttl: Duration(days: 1), + ); + final index = DartdocIndex.parseJsonText(textContent); expect(index.entries, hasLength(greaterThan(10000))); final libraries = diff --git a/tool/download-sdk-index-jsons.sh b/tool/download-sdk-index-jsons.sh new file mode 100755 index 0000000000..25110329a6 --- /dev/null +++ b/tool/download-sdk-index-jsons.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +cd "$SCRIPT_DIR/../app/" +mkdir -p .dart_tool/pub-search-data +cd .dart_tool/pub-search-data + +echo "Downloading Dart SDK index.json" +curl -o https-api.dart.dev-stable-latest-index.json https://api.dart.dev/stable/latest/index.json + +echo "Downloading Flutter SDK index.json" +curl -o https-api.flutter.dev-flutter-index.json https://api.flutter.dev/flutter/index.json