main : don't print special tokens with --grammar

The CLI interface was recently changed to print special control tokens
like the </s> stop message one. This token shouldn't be printed if the
grammar flag was passed, unless the grammar specifies it, because that
breaks shell-scriptability.

Author: jart

examples/main/main.cpp

@@ -527,6 +527,7 @@ int main(int argc, char ** argv) {
         fprintf(stderr, "%s: failed to initialize sampling subsystem\n", __func__);
         exit(1);
     }
+    bool should_show_special_tokens = sparams.grammar.empty();
 
     while ((n_remain != 0 && !is_antiprompt) || params.interactive) {
         // predict
@@ -740,7 +741,8 @@ int main(int argc, char ** argv) {
         // display text
         if (input_echo && display) {
             for (auto id : embd) {
-                const std::string token_str = llama_token_to_piece(ctx, id, !params.conversation);
+                const std::string token_str =
+                        llama_token_to_piece(ctx, id, should_show_special_tokens);
                 printf("%s", token_str.c_str());
 
                 if (embd.size() > 1) {
@@ -906,7 +908,7 @@ int main(int argc, char ** argv) {
                     for (size_t i = original_size; i < embd_inp.size(); ++i) {
                         const llama_token token = embd_inp[i];
                         output_tokens.push_back(token);
-                        output_ss << llama_token_to_piece(ctx, token);
+                        output_ss << llama_token_to_piece(ctx, token, should_show_special_tokens);
                     }
 
                     n_remain -= line_inp.size();