We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent c0956b0 commit 8960fe8Copy full SHA for 8960fe8
llama.cpp
@@ -4340,7 +4340,7 @@ static void llm_load_vocab(
4340
}
4341
4342
4343
- // find EOT token: "<|eot_id|>", "<|im_emd|>", "<end_of_turn>", etc.
+ // find EOT token: "<|eot_id|>", "<|im_end|>", "<end_of_turn>", etc.
4344
//
4345
// TODO: convert scripts should provide this token through the KV metadata LLAMA_KV_TOKENIZER_EOT_ID
4346
// for now, we apply this workaround to find the EOT token based on its text
@@ -4351,7 +4351,7 @@ static void llm_load_vocab(
4351
// need to fix convert script
4352
//vocab.id_to_token[t.second].type == LLAMA_TOKEN_TYPE_CONTROL &&
4353
(t.first == "<|eot_id|>" ||
4354
- t.first == "<|im_emd|>" ||
+ t.first == "<|im_end|>" ||
4355
t.first == "<end_of_turn>"
4356
)
4357
) {
0 commit comments