Skip to content

Commit 8960fe8

Browse files
committed
llama : fix typo in <|im_end|> token text (#6745)
1 parent c0956b0 commit 8960fe8

File tree

1 file changed

+2
-2
lines changed

1 file changed

+2
-2
lines changed

llama.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4340,7 +4340,7 @@ static void llm_load_vocab(
43404340
}
43414341
}
43424342

4343-
// find EOT token: "<|eot_id|>", "<|im_emd|>", "<end_of_turn>", etc.
4343+
// find EOT token: "<|eot_id|>", "<|im_end|>", "<end_of_turn>", etc.
43444344
//
43454345
// TODO: convert scripts should provide this token through the KV metadata LLAMA_KV_TOKENIZER_EOT_ID
43464346
// for now, we apply this workaround to find the EOT token based on its text
@@ -4351,7 +4351,7 @@ static void llm_load_vocab(
43514351
// need to fix convert script
43524352
//vocab.id_to_token[t.second].type == LLAMA_TOKEN_TYPE_CONTROL &&
43534353
(t.first == "<|eot_id|>" ||
4354-
t.first == "<|im_emd|>" ||
4354+
t.first == "<|im_end|>" ||
43554355
t.first == "<end_of_turn>"
43564356
)
43574357
) {

0 commit comments

Comments
 (0)