tools/content: Support systematically surveying unimplemented content features.

We added 2 scripts.

- fetch_messages.dart, the script that fetches messages from a given
  Zulip server, that does not depend on Flutter or other involved
  Zulip Flutter packages, so that it can run without Flutter.  It is
  meant to be run first to produce the corpuses needed for surveying
  the unimplemented features.

  The fetched messages are formatted in JSON Lines format, where each
  individual entry is JSON containing the message ID and the rendered
  HTML content.  The script stores output in separate files for
  messages from each server, because message IDs are not unique across
  them.

- unimplemented_features_test.dart, a test that goes over all messages
  collected, parses then with the content parser, and report the
  unimplemented features it discovered.  This is implemented as a test
  mainly because of its dependency on the content parser, which
  depends on the Flutter engine (and `flutter test` conveniently sets
  up a test device).

  The test can be run manually via:
    `flutter test --dart-define=corpusDir=path/to/corpusDir tools/content`

We mostly avoid prints (https://dart.dev/tools/linter-rules/avoid_print)
in both scripts.  While we don't lose much by disabling this lint rule
for them, because they are supposed CLI programs after all, the rule
(potentially) helps with reducing developer inclination to be verbose.

See comments from the scripts for more details on the implementations.

Signed-off-by: Zixuan James Li <[email protected]>

Author: PIG208

tools/content/fetch_messages.dart

@@ -0,0 +1,221 @@
+#!/usr/bin/env dart
+
+import 'dart:convert';
+import 'dart:io';
+import 'dart:math';
+
+// Avoid any Flutter-related dependencies so this can be run as a CLI program.
+import 'package:args/args.dart';
+import 'package:http/http.dart';
+import 'package:ini/ini.dart' as ini;
+import 'package:zulip/api/backoff.dart';
+
+import 'model.dart';
+
+/// Fetch all public message contents from a Zulip server in bulk.
+///
+/// It outputs JSON entries of the message IDs and the rendered HTML contents in
+/// JSON Lines (https://jsonlines.org) format. The output can be used later to
+/// perform checks for discovering unimplemented features.
+///
+/// Because message IDs are only unique within a single server, the script
+/// names corpuses from each server differently (if --corpus-dir is specified).
+///
+/// For more help, run `tools/content/fetch_message.dart --help`.
+///
+/// See also:
+/// * tools/content/unimplemented_features_test.dart, which runs checks against
+///   the fetched corpuses.
+void main(List<String> args) async {
+  final argParser = ArgParser();
+  argParser.addOption(
+    'config-file',
+    help: 'A zuliprc file with identity information including email, API key\n'
+          'and the Zulip server URL to fetch the messages from (required).\n\n'
+          'To get the file, see\n'
+          'https://zulip.com/api/configuring-python-bindings#download-a-zuliprc-file.',
+    valueHelp: 'path/to/zuliprc',
+  );
+  argParser.addOption(
+    'corpus-dir',
+    help: 'The directory to look for/store the corpus file.  If not given,\n'
+          'the script will write output to stdout.  Otherwise, this will\n'
+          'first read from the existing corpus file (assumed to be named\n'
+          'as "host name of the Zulip server.jsonl") to avoid duplicates\n'
+          'before fetching more messages',
+    valueHelp: 'path/to/czo.jsonl',
+  );
+  argParser.addFlag(
+    'fetch-newer',
+    help: 'Fetch newer messages instead of older ones.\n'
+          'Only useful when there is a matching corpus file in corpus-dir.',
+    defaultsTo: false,
+  );
+  argParser.addFlag(
+    'help', abbr: 'h',
+    negatable: false,
+    help: 'Show this help message.',
+  );
+
+  void printUsage() {
+    // Give it a pass when printing the help message.
+    // ignore: avoid_print
+    print('usage: fetch_messages --config-file <CONFIG_FILE>\n\n'
+          'Fetch message contents from a Zulip server in bulk.\n\n'
+          '${argParser.usage}');
+  }
+
+  Never throwWithUsage(String error) {
+    printUsage();
+    throw Exception('\nError: $error');
+  }
+
+  final parsedArguments = argParser.parse(args);
+  if (parsedArguments['help'] as bool) {
+    printUsage();
+    exit(0);
+  }
+
+  final zuliprc = parsedArguments['config-file'] as String?;
+  if (zuliprc == null) {
+    throwWithUsage('"config-file is required');
+  }
+
+  final configFile = File(zuliprc);
+  if (!configFile.existsSync()) {
+    throwWithUsage('Config file "$zuliprc" does not exist');
+  }
+
+  // `zuliprc` is a file in INI format containing the user's identity
+  // information.
+  //
+  // See also:
+  //   https://zulip.com/api/configuring-python-bindings#configuration-keys-and-environment-variables
+  final parsedConfig = ini.Config.fromString(configFile.readAsStringSync());
+  final email = parsedConfig.get('api', 'email') as String;
+  final apiKey = parsedConfig.get('api', 'key') as String;
+  final site = Uri.parse(parsedConfig.get('api', 'site') as String);
+
+  final outputDirStr = parsedArguments['corpus-dir'] as String?;
+  final fetchNewer = parsedArguments['fetch-newer'] as bool;
+  int? anchorMessageId;
+  IOSink output = stdout;
+  if (outputDirStr != null) {
+    final outputDir = Directory(outputDirStr);
+    outputDir.createSync(recursive: true);
+    final outputFile = File('$outputDirStr/${site.host}.jsonl');
+    if (!outputFile.existsSync()) outputFile.createSync();
+    // Look for the known newest/oldest message so that we can continue
+    // fetching from where we left off.
+    await for (final message in readMessagesFromJsonl(outputFile)) {
+      anchorMessageId ??= message.id;
+      // Newer Zulip messages have higher message IDs.
+      anchorMessageId = (fetchNewer ? max : min)(message.id, anchorMessageId);
+    }
+    output = outputFile.openWrite(mode: FileMode.writeOnlyAppend);
+  }
+
+  final client = Client();
+  final authHeader = 'Basic ${base64Encode(utf8.encode('$email:$apiKey'))}';
+
+  // These are working constants chosen abitrarily.
+  const batchSize = 5000;
+  const maxRetries = 10;
+  const fetchInterval = Duration(seconds: 5);
+
+  int retries = 0;
+  BackoffMachine? backoff;
+
+  while (true) {
+    // This loops until there is no message fetched in an iteration.
+    final _GetMessagesResult result;
+    try {
+      result = await _getMessages(client, realmUrl: site,
+        authHeader: authHeader,
+        anchorMessageId: anchorMessageId,
+        numBefore: (!fetchNewer) ? batchSize : 0,
+        numAfter: (fetchNewer) ? batchSize : 0,
+      );
+    } catch (e) {
+      // We could have more fine-grained error handling and avoid retrying on
+      // non-network-related failures, but that's not necessary.
+      if (retries >= maxRetries) {
+        rethrow;
+      }
+      retries++;
+      await (backoff ??= BackoffMachine()).wait();
+      continue;
+    }
+
+    final messageEntries = result.messages.map(MessageEntry.fromJson);
+    if (messageEntries.isEmpty) {
+      // Sanity check to ensure that the server agrees
+      // there is no more messages to fetch.
+      if (fetchNewer) assert(result.foundNewest);
+      if (!fetchNewer) assert(result.foundOldest);
+      break;
+    }
+
+    // Find and use the newest/oldest message as the next message fetch anchor.
+    anchorMessageId = messageEntries.map((x) => x.id).reduce(fetchNewer ? max : min);
+    messageEntries.map(jsonEncode).forEach((json) => output.writeln(json));
+
+    // This I/O operation could fail, but crashing is fine here.
+    final flushFuture = output.flush();
+    // Make sure the delay happens concurrently to the flush.
+    await Future<void>.delayed(fetchInterval);
+    await flushFuture;
+    backoff = null;
+  }
+  exit(0);
+}
+
+/// https://zulip.com/api/get-messages#response
+// Partially ported from [GetMessagesResult] to avoid depending on Flutter libraries.
+class _GetMessagesResult {
+  const _GetMessagesResult(this.foundOldest, this.foundNewest, this.messages);
+
+  final bool foundOldest;
+  final bool foundNewest;
+  final List<Map<String, Object?>> messages;
+
+  factory _GetMessagesResult.fromJson(Map<String, Object?> json) =>
+    _GetMessagesResult(
+      json['found_oldest'] as bool,
+      json['found_newest'] as bool,
+      (json['messages'] as List<Object?>).map((x) => (x as Map<String, Object?>)).toList());
+}
+
+/// https://zulip.com/api/get-messages
+Future<_GetMessagesResult> _getMessages(Client client, {
+  required Uri realmUrl,
+  required String authHeader,
+  required int numBefore,
+  required int numAfter,
+  int? anchorMessageId,
+}) async {
+  final url = realmUrl.replace(
+    path: '/api/v1/messages',
+    queryParameters: {
+      // This fallback will only be used when first fetching from a server.
+      'anchor': anchorMessageId != null ? jsonEncode(anchorMessageId) : 'newest',
+      // The anchor message already exists in the corpus,
+      // so avoid fetching it again.
+      'include_anchor': jsonEncode(anchorMessageId == null),
+      'num_before': jsonEncode(numBefore),
+      'num_after': jsonEncode(numAfter),
+      'narrow': jsonEncode([{'operator': 'channels', 'operand': 'public'}]),
+    });
+  final response = await client.send(
+    Request('GET', url)..headers['Authorization'] = authHeader);
+  final bytes = await response.stream.toBytes();
+  final json = jsonDecode(utf8.decode(bytes)) as Map<String, dynamic>?;
+
+  if (response.statusCode != 200 || json == null) {
+    // Just crashing early here should be fine for this tool.  We don't need
+    // to handle the specific error codes.
+    throw Exception('Failed to get messages. Code: ${response.statusCode}\n'
+                    'Details: ${json ?? 'unknown'}');
+  }
+  return _GetMessagesResult.fromJson(json);
+}

tools/content/model.dart

@@ -0,0 +1,35 @@
+import 'dart:io';
+import 'dart:convert';
+
+import 'package:json_annotation/json_annotation.dart';
+
+/// A data structure representing a message.
+@JsonSerializable()
+final class MessageEntry {
+  const MessageEntry({
+    required this.id,
+    required this.content,
+  });
+
+  /// Selectively parses from get-message responses.
+  ///
+  /// See also: https://zulip.com/api/get-messages#response
+  factory MessageEntry.fromJson(Map<String, Object?> json) =>
+      MessageEntry(id: (json['id'] as num).toInt(), content: json['content'] as String);
+
+  Map<String, Object> toJson() => {'id': id, 'content': content};
+
+  /// The message ID, unique within a server.
+  final int id;
+
+  /// The rendered HTML of the message.
+  final String content;
+}
+
+/// Open the given JSON Lines file and read [MessageEntry]'s from it.
+///
+/// We store the entries in JSON Lines format and return them from a stream to
+/// avoid excessive use of memory.
+Stream<MessageEntry> readMessagesFromJsonl(File file) => file.openRead()
+  .transform(utf8.decoder).transform(const LineSplitter())
+  .map(jsonDecode).map((x) => MessageEntry.fromJson(x as Map<String, Object?>));