vocab : add tokenizer support for jina-embeddings-v2-base-zh (#18756)

* vocab : add jina-embeddings-v2-base-zh (whitespace tokenizer)

* lowercase defaults to true

* type fix

---------

Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@scala.com>
This commit is contained in:
o7si
2026-05-31 12:37:35 +02:00
committed by GitHub
co-authored by Sigbjørn Skjæret
parent 3292da09f6
commit d4c8e2c29c
9 changed files with 106 additions and 4 deletions
+3
View File
@@ -1110,6 +1110,9 @@ class GGUFWriter:
self.add_string(Keys.Tokenizer.CHAT_TEMPLATE, value)
def add_normalizer_lowercase(self, value: bool) -> None:
self.add_bool(Keys.Tokenizer.NORMALIZER_LOWERCASE, value)
def add_eot_token_id(self, id: int) -> None:
self.add_uint32(Keys.Tokenizer.EOT_ID, id)