File tree Expand file tree Collapse file tree 2 files changed +3
-3
lines changed Expand file tree Collapse file tree 2 files changed +3
-3
lines changed Original file line number Diff line number Diff line change @@ -799,7 +799,7 @@ int main(int argc, char ** argv) {
799
799
}
800
800
801
801
const auto line_pfx = ::llama_tokenize (ctx, params.input_prefix , false , true );
802
- const auto line_inp = ::llama_tokenize (ctx, buffer, false , false );
802
+ const auto line_inp = ::llama_tokenize (ctx, buffer, false , false );
803
803
const auto line_sfx = ::llama_tokenize (ctx, params.input_suffix , false , true );
804
804
LOG (" input tokens: %s\n " , LOG_TOKENS_TOSTR_PRETTY (ctx, line_inp));
805
805
Original file line number Diff line number Diff line change @@ -2327,13 +2327,13 @@ static void llm_load_vocab(
2327
2327
}
2328
2328
2329
2329
if (special_tokens_definition_mismatch || special_tokens_count_from_verification != special_tokens_count_by_type) {
2330
- fprintf (stderr, " %s: warning: Mismatch in special tokens definition ( %u/%zu vs %u/%zu ).\n " ,
2330
+ LLAMA_LOG_WARN( "%s: mismatch in special tokens definition ( %u/%zu vs %u/%zu ).\n",
2331
2331
__func__,
2332
2332
special_tokens_count_from_verification, vocab.id_to_token.size(),
2333
2333
special_tokens_count_by_type, vocab.id_to_token.size()
2334
2334
);
2335
2335
} else {
2336
- fprintf (stderr, " %s: Special tokens definition check successful ( %u/%zu ).\n " ,
2336
+ LLAMA_LOG_INFO( "%s: special tokens definition check successful ( %u/%zu ).\n",
2337
2337
__func__,
2338
2338
special_tokens_count_from_verification, vocab.id_to_token.size()
2339
2339
);
You can’t perform that action at this time.
0 commit comments