Download SDK index.jsons at docker build. (#8699)

Author: isoos

Dockerfile.app

@@ -33,6 +33,7 @@ WORKDIR /project/app
 RUN dart /project/tool/pub_get_offline.dart /project/app
 
 RUN /project/tool/setup-webp.sh /usr/local/bin
+RUN /project/tool/download-sdk-index-jsons.sh
 
 # Clear out any arguments the base images might have set
 CMD []

app/lib/search/backend.dart

@@ -17,6 +17,7 @@ import 'package:logging/logging.dart';
 import 'package:meta/meta.dart';
 // ignore: implementation_imports
 import 'package:pana/src/dartdoc/pub_dartdoc_data.dart';
+import 'package:path/path.dart' as p;
 import 'package:pool/pool.dart';
 import 'package:pub_dev/shared/monitoring.dart';
 import 'package:retry/retry.dart';
@@ -427,17 +428,40 @@ class SearchBackend {
     }
   }
 
-  /// Downloads the remote SDK content relative to the base URI.
-  Future<String> fetchSdkIndexContentAsString({
-    required Uri baseUri,
-    required String relativePath,
+  /// Downloads the remote SDK content from [uri] and creates a cached file in the
+  /// `.dart_tool/pub-search-data/` directory.
+  ///
+  /// When a local file in `app/.dart_tool/pub-search-data/<reduced-uri>` exists,
+  /// its content will be loaded instead. URL reduction replaces slashes and other
+  /// non-characters with a single dash `-`, like:
+  /// - `https-api.dart.dev-stable-latest-index.json`
+  Future<String> loadOrFetchSdkIndexJsonAsString(
+    Uri uri, {
+    @visibleForTesting Duration? ttl,
   }) async {
-    final uri = baseUri.resolve(relativePath);
+    final fileName = uri.toString().replaceAll(RegExp(r'[^a-z0-9\.]+'), '-');
+    final file = File(p.join('.dart_tool', 'pub-search-data', fileName));
+    if (await file.exists()) {
+      var canUseCached = true;
+      if (ttl != null) {
+        final age = clock.now().difference(await file.lastModified());
+        if (age > ttl) {
+          canUseCached = false;
+        }
+      }
+      if (canUseCached) {
+        return await file.readAsString();
+      }
+    }
+
     final rs = await _http.get(uri);
     if (rs.statusCode != 200) {
       throw Exception('Unexpected status code for $uri: ${rs.statusCode}');
     }
-    return rs.body;
+    final content = rs.body;
+    await file.parent.create(recursive: true);
+    await file.writeAsString(content);
+    return content;
   }
 
   Future<List<PackageDocument>?> fetchSnapshotDocuments() async {

app/lib/search/dart_sdk_mem_index.dart

@@ -38,10 +38,8 @@ SdkMemIndex? get dartSdkMemIndex =>
 Future<SdkMemIndex?> createDartSdkMemIndex() async {
   try {
     final index = await SdkMemIndex.dart();
-    final content = await searchBackend.fetchSdkIndexContentAsString(
-      baseUri: index.baseUri,
-      relativePath: 'index.json',
-    );
+    final content =
+        await searchBackend.loadOrFetchSdkIndexJsonAsString(index.indexJsonUri);
     await index.addDartdocIndex(DartdocIndex.parseJsonText(content));
     index.updateWeights(
       libraryWeights: dartSdkLibraryWeights,

app/lib/search/flutter_sdk_mem_index.dart

@@ -55,10 +55,8 @@ SdkMemIndex? get flutterSdkMemIndex =>
 Future<SdkMemIndex?> createFlutterSdkMemIndex() async {
   try {
     final index = SdkMemIndex.flutter();
-    final content = await searchBackend.fetchSdkIndexContentAsString(
-      baseUri: index.baseUri,
-      relativePath: 'index.json',
-    );
+    final content =
+        await searchBackend.loadOrFetchSdkIndexJsonAsString(index.indexJsonUri);
     await index.addDartdocIndex(DartdocIndex.parseJsonText(content),
         allowedLibraries: _allowedLibraries);
     index.updateWeights(

app/lib/search/sdk_mem_index.dart

@@ -4,7 +4,6 @@
 
 import 'dart:math';
 
-import 'package:_pub_shared/utils/http.dart';
 import 'package:meta/meta.dart';
 // ignore: implementation_imports
 import 'package:pana/src/dartdoc/dartdoc_index.dart';
@@ -36,39 +35,13 @@ class SdkMemIndex {
         _baseUri = baseUri;
 
   static Future<SdkMemIndex> dart() async {
-    final versions = <String>{
-      toolStableDartSdkVersion,
-      runtimeSdkVersion,
-    };
-    final client = httpRetryClient();
-    try {
-      for (final version in versions) {
-        final uri = _dartSdkBaseUri(version);
-        final rs = await client.head(uri);
-        if (rs.statusCode < 400) {
-          return SdkMemIndex(sdk: 'dart', version: version, baseUri: uri);
-        }
-      }
-    } finally {
-      client.close();
-    }
     return SdkMemIndex(
       sdk: 'dart',
       version: runtimeSdkVersion,
-      baseUri: _dartSdkBaseUri(runtimeSdkVersion),
+      baseUri: Uri.parse('https://api.dart.dev/stable/latest/'),
     );
   }
 
-  static Uri _dartSdkBaseUri(String version) {
-    var branch = 'stable';
-    if (version.contains('beta')) {
-      branch = 'beta';
-    } else if (version.contains('dev')) {
-      branch = 'dev';
-    }
-    return Uri.parse('https://api.dart.dev/$branch/$version/');
-  }
-
   factory SdkMemIndex.flutter() {
     return SdkMemIndex(
       sdk: 'flutter',
@@ -77,7 +50,7 @@ class SdkMemIndex {
     );
   }
 
-  Uri get baseUri => _baseUri;
+  late final indexJsonUri = _baseUri.resolve('index.json');
 
   Future<void> addDartdocIndex(
     DartdocIndex index, {

app/test/search/backend_test.dart

@@ -14,10 +14,8 @@ void main() {
   group('search backend', () {
     testWithProfile('fetch SDK library description', fn: () async {
       final index = await SdkMemIndex.dart();
-      final content = await searchBackend.fetchSdkIndexContentAsString(
-        baseUri: index.baseUri,
-        relativePath: 'index.json',
-      );
+      final content = await searchBackend
+          .loadOrFetchSdkIndexJsonAsString(index.indexJsonUri);
       await index.addDartdocIndex(DartdocIndex.parseJsonText(content));
       expect(
         index.getLibraryDescription('dart:async'),

app/test/search/dartdoc_index_parsing_test.dart

@@ -3,46 +3,21 @@
 // BSD-style license that can be found in the LICENSE file.
 
 import 'dart:convert';
-import 'dart:io';
 
-import 'package:clock/clock.dart';
-import 'package:http/http.dart' as http;
-import 'package:path/path.dart' as p;
+import 'package:pub_dev/search/backend.dart';
 import 'package:pub_dev/search/sdk_mem_index.dart';
-import 'package:pub_dev/shared/versions.dart';
-import 'package:retry/retry.dart';
 import 'package:test/test.dart';
 
+import '../shared/test_services.dart';
+
 void main() {
   group('dartdoc index.json parsing', () {
-    /// Downloads [url and creates a cached file in the .dart_tool/pub-search-data directory.
-    ///
-    /// Reuses the same file up to a week.
-    Future<File> getCachedFile(String name, String url) async {
-      final file = File(p.join('.dart_tool', 'pub-search-data', name));
-      if (await file.exists()) {
-        final lastModified = await file.lastModified();
-        final age = clock.now().difference(lastModified);
-        if (age.inDays < 7) {
-          return file;
-        }
-      }
-      await file.parent.create(recursive: true);
-      return retry(() async {
-        final rs = await http.get(Uri.parse(url));
-        if (rs.statusCode != 200) {
-          throw Exception('Unexpected status code for $url: ${rs.statusCode}');
-        }
-        await file.writeAsBytes(rs.bodyBytes);
-        return file;
-      });
-    }
-
-    test('parse Dart SDK index.json', () async {
-      final file = await getCachedFile('dart-sdk-$runtimeSdkVersion.json',
-          'https://api.dart.dev/stable/$runtimeSdkVersion/index.json');
-      final textContent = await file.readAsString();
-      final index = DartdocIndex.parseJsonText(await file.readAsString());
+    testWithProfile('parse Dart SDK index.json', fn: () async {
+      final textContent = await searchBackend.loadOrFetchSdkIndexJsonAsString(
+        Uri.parse('https://api.dart.dev/stable/latest/index.json'),
+        ttl: Duration(days: 1),
+      );
+      final index = DartdocIndex.parseJsonText(textContent);
       expect(index.entries, hasLength(greaterThan(10000)));
 
       final libraries =
@@ -59,11 +34,12 @@ void main() {
       expect(json.decode(index.toJsonText()), json.decode(textContent));
     });
 
-    test('parse Flutter SDK index.json', () async {
-      final file = await getCachedFile(
-          'flutter-sdk.json', 'https://api.flutter.dev/flutter/index.json');
-      final textContent = await file.readAsString();
-      final index = DartdocIndex.parseJsonText(await file.readAsString());
+    testWithProfile('parse Flutter SDK index.json', fn: () async {
+      final textContent = await searchBackend.loadOrFetchSdkIndexJsonAsString(
+        Uri.parse('https://api.flutter.dev/flutter/index.json'),
+        ttl: Duration(days: 1),
+      );
+      final index = DartdocIndex.parseJsonText(textContent);
       expect(index.entries, hasLength(greaterThan(10000)));
 
       final libraries =

tool/download-sdk-index-jsons.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -e
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+cd "$SCRIPT_DIR/../app/"
+mkdir -p .dart_tool/pub-search-data
+cd .dart_tool/pub-search-data
+
+echo "Downloading Dart SDK index.json"
+curl -o https-api.dart.dev-stable-latest-index.json https://api.dart.dev/stable/latest/index.json
+
+echo "Downloading Flutter SDK index.json"
+curl -o https-api.flutter.dev-flutter-index.json https://api.flutter.dev/flutter/index.json