tools/check: Add suite (message content) parser.

Some main benefits of adding both tools/content scripts to tools/check
are that you don't need to manage your directories for storing the
zuliprc files and corpuses, or specify any command line options for
fetching message contents and running the unimplemented features test.

The script is intended to be run manually, not as a part of the CI,
because it is very slow, and it relies on some out of tree files like
API configs (zuliprc files) and big dumps of chat history.

Signed-off-by: Zixuan James Li <[email protected]>

Author: PIG208

.gitignore

@@ -53,3 +53,7 @@ app.*.map.json
 
 # Old scaffolding hack
 lib/credential_fixture.dart
+
+# Directories used by `tools/check parser`
+.corpuses
+.api_configs

tools/check

@@ -82,7 +82,7 @@ while (( $# )); do
         --all) opt_files=all; opt_all=1; shift;;
         --fix) opt_fix=1; shift;;
         --verbose) opt_verbose=1; shift;;
-        analyze|test|build_runner|drift|pigeon|icons|android|shellcheck)
+        analyze|test|build_runner|drift|pigeon|icons|android|parser|shellcheck)
             opt_suites+=("$1"); shift;;
         *) usage;;
     esac
@@ -386,6 +386,51 @@ run_android() {
     flutter build appbundle
 }
 
+run_parser() {
+    # Needed to skip fetching messages when there is no match for the
+    # `config_files` glob.
+    shopt -s nullglob
+    local config_dir=.api_configs
+    local config_files=("$config_dir"/*)
+
+    local corpus_dir=.corpuses
+    local corpus_files=("$corpus_dir"/*)
+
+    # `tools/content/fetch_messages.dart` will create corpus files in
+    # `corpus_dir` for each host from the zuliprc files, which is needed for
+    # running `tools/content/unimplemented_features_test`.
+    for config in "${config_files[@]}"; do
+        echo "Fetching all public messages using API config \"$config\"."\
+             "This can take a long time."
+        tools/content/fetch_messages.dart --config-file "$config" \
+            --corpus-dir $corpus_dir \
+        || return 1
+    done
+
+    if (( ! "${#config_files[@]}" )) && (( ! ${#corpus_files[@]} )); then
+        cat >&2 <<EOF
+tools/check parser: bad configuration
+
+Running checks for unimplemented features require either zuliprc files in
+$config_dir to fetch corpuses or existing corpus files in $corpus_dir,
+but both directories are found empty.
+
+Consider downloading zuliprc to $config_dir:
+  https://zulip.com/api/configuring-python-bindings#download-a-zuliprc-file
+or copying pre-fetched corpuses to $corpus_dir.
+EOF
+        return 1
+    fi
+
+    # This test fails on a fresh clone that do not have .api_configs or
+    # .corpus set up.  This is intentional because it is supposed to be run
+    # manually with some API keys and/or corpuses prepared.
+    flutter test tools/content/unimplemented_features_test.dart \
+        --dart-define=corpusDir=$corpus_dir \
+        --dart-define=verbose=$opt_verbose \
+    || return 1
+}
+
 run_shellcheck() {
     # Omitted from this check: nothing (nothing known, anyway).
     files_check tools/ '!*.'{dart,js,json} \
@@ -460,6 +505,7 @@ for suite in "${opt_suites[@]}"; do
         pigeon)       run_pigeon ;;
         icons)        run_icons ;;
         android)      run_android ;;
+        parser)       run_parser ;;
         shellcheck)   run_shellcheck ;;
         *)            echo >&2 "Internal error: unknown suite $suite" ;;
     esac || failed+=( "$suite" )