Migrating pub_server: versions list API. (#3508)

* Migrating pub_server: versions list API.

* code review updates

Author: isoos

app/lib/frontend/handlers/pubapi.client.dart

@@ -29,8 +29,8 @@ class PubApi {
   /// Getting information about all versions of a package.
   /// https://github.com/dart-lang/pub/blob/master/doc/repository-spec-v2.md#list-all-versions-of-a-package
   @EndPoint.get('/api/packages/<package>')
-  Future<Response> listPackageVersions(Request request, String package) async =>
-      await packageBackend.pubServer
+  Future<PackageData> packageData(Request request, String package) async =>
+      await packageBackend.repository
           .listVersions(_replaceHost(request.requestedUri), package);
 
   /// Getting information about a specific (package, version) pair.

app/lib/frontend/handlers/pubapi.dart

@@ -377,14 +377,29 @@ class GCloudPackageRepository extends pub_server.PackageRepository {
 
   GCloudPackageRepository(this.db, this.storage);
 
-  // Metadata support.
-
-  @override
-  Stream<pub_server.PackageVersion> versions(String package) {
-    final packageKey = db.emptyKey.append(Package, id: package);
-    final query = db.query<PackageVersion>(ancestorKey: packageKey);
-    return query.run().map((model) => pub_server.PackageVersion(
-        package, model.version, model.pubspec.jsonString));
+  /// Returns the known versions of [package].
+  ///
+  /// Used in `pub` client for finding which versions exist.
+  Future<api.PackageData> listVersions(Uri baseUri, String package) async {
+    return await cache.packageData(package).get(() async {
+      final packageVersions = await packageBackend.versionsOfPackage(package);
+      if (packageVersions.isEmpty) {
+        throw NotFoundException.resource('package "$package"');
+      }
+      packageVersions
+          .sort((a, b) => a.semanticVersion.compareTo(b.semanticVersion));
+      final latest = packageVersions.lastWhere(
+        (pv) => !pv.semanticVersion.isPreRelease,
+        orElse: () => packageVersions.last,
+      );
+      return api.PackageData(
+        name: package,
+        latest: _toApiVersionInfo(baseUri, latest),
+        versions: packageVersions
+            .map((pv) => _toApiVersionInfo(baseUri, pv))
+            .toList(),
+      );
+    });
   }
 
   /// Lookup and return the API's version info object.
@@ -405,14 +420,17 @@ class GCloudPackageRepository extends pub_server.PackageRepository {
       throw NotFoundException.resource('version "$version"');
     }
 
-    return api.VersionInfo(
-      version: pv.version,
-      pubspec: pv.pubspec.asJson,
-      archiveUrl:
-          urls.pkgArchiveDownloadUrl(pv.package, pv.version, baseUri: baseUri),
-    );
+    return _toApiVersionInfo(baseUri, pv);
   }
 
+  api.VersionInfo _toApiVersionInfo(Uri baseUri, PackageVersion pv) =>
+      api.VersionInfo(
+        version: pv.version,
+        pubspec: pv.pubspec.asJson,
+        archiveUrl: urls.pkgArchiveDownloadUrl(pv.package, pv.version,
+            baseUri: baseUri),
+      );
+
   // Download support.
 
   @override

app/lib/frontend/handlers/pubapi.g.dart

@@ -50,9 +50,6 @@ class PackageVersion {
 
 /// Represents a pub repository.
 abstract class PackageRepository {
-  /// Returns the known versions of [package].
-  Stream<PackageVersion> versions(String package);
-
   /// Starts a  upload.
   ///
   /// The given [redirectUrl] instructs the uploading client to make a GET

app/lib/package/backend.dart

@@ -9,12 +9,10 @@ import 'dart:convert' as convert;
 
 import 'package:logging/logging.dart';
 import 'package:shelf/shelf.dart' as shelf;
-import 'package:yaml/yaml.dart';
 
 import '../../package/backend.dart' show purgePackageCache;
 import '../../shared/exceptions.dart';
 import '../../shared/redis_cache.dart' show cache;
-import '../../shared/urls.dart' as urls;
 
 import 'repository.dart';
 
@@ -27,51 +25,6 @@ class ShelfPubServer {
 
   ShelfPubServer(this.repository);
 
-  // Metadata handlers.
-
-  Future<shelf.Response> listVersions(Uri uri, String package) async {
-    final cachedBinaryJson = await cache.packageData(package).get();
-    if (cachedBinaryJson != null) {
-      return _binaryJsonResponse(cachedBinaryJson);
-    }
-
-    final packageVersions = await repository.versions(package).toList();
-    if (packageVersions.isEmpty) {
-      return shelf.Response.notFound(null);
-    }
-
-    packageVersions.sort((a, b) => a.version.compareTo(b.version));
-
-    // TODO: Add legacy entries (if necessary), such as version_url.
-    Map packageVersion2Json(PackageVersion version) {
-      return {
-        'archive_url': urls.pkgArchiveDownloadUrl(
-            version.packageName, version.versionString,
-            baseUri: uri),
-        'pubspec': loadYaml(version.pubspecYaml),
-        'version': version.versionString,
-      };
-    }
-
-    var latestVersion = packageVersions.last;
-    for (int i = packageVersions.length - 1; i >= 0; i--) {
-      if (!packageVersions[i].version.isPreRelease) {
-        latestVersion = packageVersions[i];
-        break;
-      }
-    }
-
-    // TODO: The 'latest' is something we should get rid of, since it's
-    // duplicated in 'versions'.
-    final binaryJson = convert.json.encoder.fuse(convert.utf8.encoder).convert({
-      'name': package,
-      'latest': packageVersion2Json(latestVersion),
-      'versions': packageVersions.map(packageVersion2Json).toList(),
-    });
-    await cache.packageData(package).set(binaryJson);
-    return _binaryJsonResponse(binaryJson);
-  }
-
   // Upload async handlers.
 
   Future<shelf.Response> finishUploadAsync(Uri uri) async {
@@ -102,11 +55,6 @@ class ShelfPubServer {
 
   // Helper functions.
 
-  shelf.Response _binaryJsonResponse(List<int> d, {int status = 200}) =>
-      shelf.Response(status,
-          body: Stream.fromIterable([d]),
-          headers: {'content-type': 'application/json'});
-
   shelf.Response _jsonResponse(Map json, {int status = 200}) =>
       shelf.Response(status,
           body: convert.json.encode(json),

app/lib/package/pub_server/repository.dart

@@ -6,10 +6,11 @@ import 'dart:async';
 import 'dart:convert';
 import 'dart:io';
 
-import 'package:neat_cache/neat_cache.dart';
-import 'package:gcloud/service_scope.dart' as ss;
 import 'package:appengine/appengine.dart';
+import 'package:client_data/package_api.dart' show PackageData;
+import 'package:gcloud/service_scope.dart' as ss;
 import 'package:logging/logging.dart';
+import 'package:neat_cache/neat_cache.dart';
 
 import '../account/models.dart' show LikeData, UserSessionData;
 import '../dartdoc/models.dart' show DartdocEntry, FileInfo;
@@ -113,9 +114,15 @@ class CachePatterns {
       .withTTL(Duration(minutes: 60))
       .withCodec(utf8)[publisherId];
 
-  Entry<List<int>> packageData(String package) => _cache
-      .withPrefix('package-data')
-      .withTTL(Duration(minutes: 10))['$package'];
+  Entry<PackageData> packageData(String package) => _cache
+      .withPrefix('api-package-data')
+      .withTTL(Duration(minutes: 10))
+      .withCodec(utf8)
+      .withCodec(json)
+      .withCodec(wrapAsCodec(
+        encode: (PackageData pd) => pd.toJson(),
+        decode: (d) => PackageData.fromJson(d as Map<String, dynamic>),
+      ))['$package'];
 
   Entry<PackageView> packageView(String package) => _cache
       .withPrefix('package-view')

app/lib/package/pub_server/shelf_pubserver.dart

@@ -347,22 +347,23 @@ void main() {
       });
     });
 
-    group('GCloudRepository.versions', () {
+    group('GCloudRepository.packageData', () {
+      final baseUri = Uri.parse('https://pub.dev');
+
       testWithServices('not found', () async {
-        final versions =
-            await packageBackend.repository.versions('non_hydrogen').toList();
-        expect(versions, isEmpty);
+        final rs =
+            packageBackend.repository.listVersions(baseUri, 'non_hydrogen');
+        await expectLater(rs, throwsA(isA<NotFoundException>()));
       });
 
       testWithServices('found', () async {
-        final versions =
-            await packageBackend.repository.versions('hydrogen').toList();
-        expect(versions, isNotEmpty);
-        expect(versions, hasLength(13));
-        expect(versions.first.packageName, 'hydrogen');
-        expect(versions.first.versionString, '1.0.0');
-        expect(versions.last.packageName, 'hydrogen');
-        expect(versions.last.versionString, '2.0.8');
+        final pd =
+            await packageBackend.repository.listVersions(baseUri, 'hydrogen');
+        expect(pd.versions, isNotEmpty);
+        expect(pd.versions, hasLength(13));
+        expect(pd.versions.first.version, '1.0.0');
+        expect(pd.versions.last.version, '2.0.8');
+        expect(pd.latest.version, '2.0.8');
       });
     });
 

app/lib/shared/redis_cache.dart

@@ -88,12 +88,42 @@ class Message {
   Map<String, dynamic> toJson() => _$MessageToJson(this);
 }
 
+/// Used in `pub` client for finding which versions exist.
+/// (`listVersions` method in pubapi)
+@JsonSerializable()
+class PackageData {
+  /// Package name.
+  final String name;
+
+  /// This is merely a convenience property, because the [VersionInfo] for the
+  /// latest version also exists in the [versions] list.
+  ///
+  /// This is the latest that is NOT a pre-release, unless there only is
+  /// pre-releases in the [versions] list.
+  final VersionInfo latest;
+
+  /// The available versions, sorted by their semantic version number (ascending).
+  final List<VersionInfo> versions;
+
+  PackageData({
+    @required this.name,
+    @required this.latest,
+    @required this.versions,
+  });
+
+  factory PackageData.fromJson(Map<String, dynamic> json) =>
+      _$PackageDataFromJson(json);
+
+  Map<String, dynamic> toJson() => _$PackageDataToJson(this);
+}
+
 @JsonSerializable()
 class VersionInfo {
   final String version;
 
   final Map<String, dynamic> pubspec;
 
+  /// As of Dart 2.8 `pub` client uses [archiveUrl] to find the archive.
   @JsonKey(name: 'archive_url')
   final String archiveUrl;