vocab : add tokenizer support for jina-embeddings-v2-base-zh (#18756)

* vocab : add jina-embeddings-v2-base-zh (whitespace tokenizer)

* lowercase defaults to true

* type fix

---------

Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@scala.com>
This commit is contained in:
o7si
2026-05-31 12:37:35 +02:00
committed by GitHub
co-authored by Sigbjørn Skjæret
parent 3292da09f6
commit d4c8e2c29c
9 changed files with 106 additions and 4 deletions
+2
View File
@@ -268,6 +268,8 @@ class Keys:
CHAT_TEMPLATE = "tokenizer.chat_template"
CHAT_TEMPLATE_N = "tokenizer.chat_template.{name}"
CHAT_TEMPLATES = "tokenizer.chat_templates"
# Normalizer constants
NORMALIZER_LOWERCASE = "tokenizer.ggml.normalizer.lowercase"
# FIM/Infill special tokens constants
FIM_PRE_ID = "tokenizer.ggml.fim_pre_token_id"
FIM_SUF_ID = "tokenizer.ggml.fim_suf_token_id"