File tree Expand file tree Collapse file tree 1 file changed +2
-2
lines changed
Expand file tree Collapse file tree 1 file changed +2
-2
lines changed Original file line number Diff line number Diff line change @@ -4340,7 +4340,7 @@ static void llm_load_vocab(
43404340 }
43414341 }
43424342
4343- // find EOT token: "<|eot_id|>", "<|im_emd |>", "<end_of_turn>", etc.
4343+ // find EOT token: "<|eot_id|>", "<|im_end |>", "<end_of_turn>", etc.
43444344 //
43454345 // TODO: convert scripts should provide this token through the KV metadata LLAMA_KV_TOKENIZER_EOT_ID
43464346 // for now, we apply this workaround to find the EOT token based on its text
@@ -4351,7 +4351,7 @@ static void llm_load_vocab(
43514351 // need to fix convert script
43524352 //vocab.id_to_token[t.second].type == LLAMA_TOKEN_TYPE_CONTROL &&
43534353 (t.first == "<|eot_id|>" ||
4354- t.first == "<|im_emd |>" ||
4354+ t.first == "<|im_end |>" ||
43554355 t.first == "<end_of_turn>"
43564356 )
43574357 ) {
You can’t perform that action at this time.
0 commit comments