nikola/llama_cpp
1
0
Fork 0
Code Issues Pull Requests Actions 23 Packages Projects Releases Wiki Activity

vocab : refactor normalizer flags into options struct, add strip_accents (#24371)

Browse Source 
* vocab : refactor normalizer flags into options struct, add strip_accents

* Update src/llama-vocab.h

Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@scala.com>

* Update src/llama-vocab.cpp

Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@scala.com>

---------

Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@scala.com>
This commit is contained in:
o7si
2026-06-11 10:36:50 +03:00
committed by GitHub
co-authored by Sigbjørn Skjæret
parent db94854ff5
commit 68f30663cf
7 changed files with 84 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gguf-py/gguf/constants.py
+2 -1
View File
@@ -272,7 +272,8 @@ class Keys:
CHAT_TEMPLATE_N = "tokenizer.chat_template.{name}" CHAT_TEMPLATE_N = "tokenizer.chat_template.{name}"
CHAT_TEMPLATES = "tokenizer.chat_templates" CHAT_TEMPLATES = "tokenizer.chat_templates"
# Normalizer constants # Normalizer constants
NORMALIZER_LOWERCASE = "tokenizer.ggml.normalizer.lowercase" NORMALIZER_LOWERCASE = "tokenizer.ggml.normalizer.lowercase"
NORMALIZER_STRIP_ACCENTS = "tokenizer.ggml.normalizer.strip_accents"
# FIM/Infill special tokens constants # FIM/Infill special tokens constants
FIM_PRE_ID = "tokenizer.ggml.fim_pre_token_id" FIM_PRE_ID = "tokenizer.ggml.fim_pre_token_id"
FIM_SUF_ID = "tokenizer.ggml.fim_suf_token_id" FIM_SUF_ID = "tokenizer.ggml.fim_suf_token_id"
gguf-py/gguf/gguf_writer.py
+3
View File
@@ -1124,6 +1124,9 @@ class GGUFWriter:
def add_normalizer_lowercase(self, value: bool) -> None: def add_normalizer_lowercase(self, value: bool) -> None:
self.add_bool(Keys.Tokenizer.NORMALIZER_LOWERCASE, value) self.add_bool(Keys.Tokenizer.NORMALIZER_LOWERCASE, value)
def add_normalizer_strip_accents(self, value: bool) -> None:
self.add_bool(Keys.Tokenizer.NORMALIZER_STRIP_ACCENTS, value)
def add_eot_token_id(self, id: int) -> None: def add_eot_token_id(self, id: int) -> None:
self.add_uint32(Keys.Tokenizer.EOT_ID, id) self.add_uint32(Keys.Tokenizer.EOT_ID, id)
gguf-py/gguf/vocab.py
+14 -4
View File
@@ -53,6 +53,7 @@ class SpecialVocab:
special_token_ids: dict[str, int] special_token_ids: dict[str, int]
chat_template: str | Sequence[Mapping[str, str]] | None chat_template: str | Sequence[Mapping[str, str]] | None
normalizer_lowercase: bool | None normalizer_lowercase: bool | None
normalizer_strip_accents: bool | None
def __init__( def __init__(
self, path: str | os.PathLike[str], load_merges: bool = False, self, path: str | os.PathLike[str], load_merges: bool = False,
@@ -66,6 +67,7 @@ class SpecialVocab:
self.merges = [] self.merges = []
self.chat_template = None self.chat_template = None
self.normalizer_lowercase = None self.normalizer_lowercase = None
self.normalizer_strip_accents = None
if special_token_types is not None: if special_token_types is not None:
self.special_token_types = special_token_types self.special_token_types = special_token_types
else: else:
@@ -108,6 +110,10 @@ class SpecialVocab:
if not quiet: if not quiet:
logger.info(f'Setting normalizer_lowercase to {self.normalizer_lowercase}') logger.info(f'Setting normalizer_lowercase to {self.normalizer_lowercase}')
gw.add_normalizer_lowercase(self.normalizer_lowercase) gw.add_normalizer_lowercase(self.normalizer_lowercase)
if self.normalizer_strip_accents is not None:
if not quiet:
logger.info(f'Setting normalizer_strip_accents to {self.normalizer_strip_accents}')
gw.add_normalizer_strip_accents(self.normalizer_strip_accents)
def _load(self, path: Path) -> None: def _load(self, path: Path) -> None:
self._try_load_from_tokenizer_json(path) self._try_load_from_tokenizer_json(path)
@@ -155,17 +161,21 @@ class SpecialVocab:
def _parse_normalizer(self, normalizer: dict) -> None: def _parse_normalizer(self, normalizer: dict) -> None:
# ref: https://huggingface.co/docs/tokenizers/api/normalizers # ref: https://huggingface.co/docs/tokenizers/api/normalizers
# #
# Detects lowercase normalization in three possible formats: # Extracts normalizer flags from three possible formats:
# 1. Standalone: {"type": "Lowercase"} # 1. Standalone: {"type": "Lowercase"}
# 2. BertNormalizer attribute: {"type": "BertNormalizer", "lowercase": true, ...} # 2. BertNormalizer attrs: {"type": "BertNormalizer", ...}
# 3. Nested in Sequence: {"type": "Sequence", "normalizers": [...]} # 3. Nested in Sequence: {"type": "Sequence", "normalizers": [...]}
normalizer_type = normalizer.get('type') normalizer_type = normalizer.get('type')
if normalizer_type == 'Lowercase': if normalizer_type == 'Lowercase':
self.normalizer_lowercase = True self.normalizer_lowercase = True
elif normalizer_type == 'StripAccents':
self.normalizer_strip_accents = True
elif normalizer_type == 'BertNormalizer': elif normalizer_type == 'BertNormalizer':
if 'lowercase' in normalizer: if 'lowercase' in normalizer:
self.normalizer_lowercase = normalizer['lowercase'] self.normalizer_lowercase = normalizer['lowercase']
if 'strip_accents' in normalizer:
self.normalizer_strip_accents = normalizer['strip_accents']
elif normalizer_type == 'Sequence': elif normalizer_type == 'Sequence':
for norm in normalizer.get('normalizers', []): for norm in normalizer.get('normalizers', []):
self._parse_normalizer(norm) self._parse_normalizer(norm)
src/llama-arch.cpp
+34 -33
View File
@@ -299,39 +299,40 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
{ LLM_KV_DENSE_3_FEAT_IN, "%s.dense_3_feat_in" }, { LLM_KV_DENSE_3_FEAT_IN, "%s.dense_3_feat_in" },
{ LLM_KV_DENSE_3_FEAT_OUT, "%s.dense_3_feat_out" }, { LLM_KV_DENSE_3_FEAT_OUT, "%s.dense_3_feat_out" },
{ LLM_KV_TOKENIZER_MODEL, "tokenizer.ggml.model" }, { LLM_KV_TOKENIZER_MODEL, "tokenizer.ggml.model" },
{ LLM_KV_TOKENIZER_PRE, "tokenizer.ggml.pre" }, { LLM_KV_TOKENIZER_PRE, "tokenizer.ggml.pre" },
{ LLM_KV_TOKENIZER_LIST, "tokenizer.ggml.tokens" }, { LLM_KV_TOKENIZER_LIST, "tokenizer.ggml.tokens" },
{ LLM_KV_TOKENIZER_TOKEN_TYPE, "tokenizer.ggml.token_type" }, { LLM_KV_TOKENIZER_TOKEN_TYPE, "tokenizer.ggml.token_type" },
{ LLM_KV_TOKENIZER_TOKEN_TYPE_COUNT, "tokenizer.ggml.token_type_count" }, { LLM_KV_TOKENIZER_TOKEN_TYPE_COUNT, "tokenizer.ggml.token_type_count" },
{ LLM_KV_TOKENIZER_SCORES, "tokenizer.ggml.scores" }, { LLM_KV_TOKENIZER_SCORES, "tokenizer.ggml.scores" },
{ LLM_KV_TOKENIZER_MERGES, "tokenizer.ggml.merges" }, { LLM_KV_TOKENIZER_MERGES, "tokenizer.ggml.merges" },
{ LLM_KV_TOKENIZER_BOS_ID, "tokenizer.ggml.bos_token_id" }, { LLM_KV_TOKENIZER_BOS_ID, "tokenizer.ggml.bos_token_id" },
{ LLM_KV_TOKENIZER_EOS_ID, "tokenizer.ggml.eos_token_id" }, { LLM_KV_TOKENIZER_EOS_ID, "tokenizer.ggml.eos_token_id" },
{ LLM_KV_TOKENIZER_EOT_ID, "tokenizer.ggml.eot_token_id" }, { LLM_KV_TOKENIZER_EOT_ID, "tokenizer.ggml.eot_token_id" },
{ LLM_KV_TOKENIZER_EOM_ID, "tokenizer.ggml.eom_token_id" }, { LLM_KV_TOKENIZER_EOM_ID, "tokenizer.ggml.eom_token_id" },
{ LLM_KV_TOKENIZER_UNK_ID, "tokenizer.ggml.unknown_token_id" }, { LLM_KV_TOKENIZER_UNK_ID, "tokenizer.ggml.unknown_token_id" },
{ LLM_KV_TOKENIZER_SEP_ID, "tokenizer.ggml.seperator_token_id" }, { LLM_KV_TOKENIZER_SEP_ID, "tokenizer.ggml.seperator_token_id" },
{ LLM_KV_TOKENIZER_PAD_ID, "tokenizer.ggml.padding_token_id" }, { LLM_KV_TOKENIZER_PAD_ID, "tokenizer.ggml.padding_token_id" },
{ LLM_KV_TOKENIZER_CLS_ID, "tokenizer.ggml.cls_token_id" }, { LLM_KV_TOKENIZER_CLS_ID, "tokenizer.ggml.cls_token_id" },
{ LLM_KV_TOKENIZER_MASK_ID, "tokenizer.ggml.mask_token_id" }, { LLM_KV_TOKENIZER_MASK_ID, "tokenizer.ggml.mask_token_id" },
{ LLM_KV_TOKENIZER_ADD_BOS, "tokenizer.ggml.add_bos_token" }, { LLM_KV_TOKENIZER_ADD_BOS, "tokenizer.ggml.add_bos_token" },
{ LLM_KV_TOKENIZER_ADD_EOS, "tokenizer.ggml.add_eos_token" }, { LLM_KV_TOKENIZER_ADD_EOS, "tokenizer.ggml.add_eos_token" },
{ LLM_KV_TOKENIZER_ADD_SEP, "tokenizer.ggml.add_sep_token" }, { LLM_KV_TOKENIZER_ADD_SEP, "tokenizer.ggml.add_sep_token" },
{ LLM_KV_TOKENIZER_ADD_PREFIX, "tokenizer.ggml.add_space_prefix" }, { LLM_KV_TOKENIZER_ADD_PREFIX, "tokenizer.ggml.add_space_prefix" },
{ LLM_KV_TOKENIZER_REMOVE_EXTRA_WS, "tokenizer.ggml.remove_extra_whitespaces" }, { LLM_KV_TOKENIZER_REMOVE_EXTRA_WS, "tokenizer.ggml.remove_extra_whitespaces" },
{ LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP, "tokenizer.ggml.precompiled_charsmap" }, { LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP, "tokenizer.ggml.precompiled_charsmap" },
{ LLM_KV_TOKENIZER_HF_JSON, "tokenizer.huggingface.json" }, { LLM_KV_TOKENIZER_HF_JSON, "tokenizer.huggingface.json" },
{ LLM_KV_TOKENIZER_RWKV, "tokenizer.rwkv.world" }, { LLM_KV_TOKENIZER_RWKV, "tokenizer.rwkv.world" },
{ LLM_KV_TOKENIZER_CHAT_TEMPLATE, "tokenizer.chat_template" }, { LLM_KV_TOKENIZER_CHAT_TEMPLATE, "tokenizer.chat_template" },
{ LLM_KV_TOKENIZER_NORMALIZER_LOWERCASE, "tokenizer.ggml.normalizer.lowercase" }, { LLM_KV_TOKENIZER_NORMALIZER_LOWERCASE, "tokenizer.ggml.normalizer.lowercase" },
{ LLM_KV_TOKENIZER_FIM_PRE_ID, "tokenizer.ggml.fim_pre_token_id" }, { LLM_KV_TOKENIZER_NORMALIZER_STRIP_ACCENTS, "tokenizer.ggml.normalizer.strip_accents" },
{ LLM_KV_TOKENIZER_FIM_SUF_ID, "tokenizer.ggml.fim_suf_token_id" }, { LLM_KV_TOKENIZER_FIM_PRE_ID, "tokenizer.ggml.fim_pre_token_id" },
{ LLM_KV_TOKENIZER_FIM_MID_ID, "tokenizer.ggml.fim_mid_token_id" }, { LLM_KV_TOKENIZER_FIM_SUF_ID, "tokenizer.ggml.fim_suf_token_id" },
{ LLM_KV_TOKENIZER_FIM_PAD_ID, "tokenizer.ggml.fim_pad_token_id" }, { LLM_KV_TOKENIZER_FIM_MID_ID, "tokenizer.ggml.fim_mid_token_id" },
{ LLM_KV_TOKENIZER_FIM_REP_ID, "tokenizer.ggml.fim_rep_token_id" }, { LLM_KV_TOKENIZER_FIM_PAD_ID, "tokenizer.ggml.fim_pad_token_id" },
{ LLM_KV_TOKENIZER_FIM_SEP_ID, "tokenizer.ggml.fim_sep_token_id" }, { LLM_KV_TOKENIZER_FIM_REP_ID, "tokenizer.ggml.fim_rep_token_id" },
{ LLM_KV_TOKENIZER_SUPPRESS_TOKENS, "tokenizer.ggml.suppress_tokens" }, { LLM_KV_TOKENIZER_FIM_SEP_ID, "tokenizer.ggml.fim_sep_token_id" },
{ LLM_KV_TOKENIZER_SUPPRESS_TOKENS, "tokenizer.ggml.suppress_tokens" },
{ LLM_KV_ADAPTER_TYPE, "adapter.type" }, { LLM_KV_ADAPTER_TYPE, "adapter.type" },
{ LLM_KV_ADAPTER_LORA_ALPHA, "adapter.lora.alpha" }, { LLM_KV_ADAPTER_LORA_ALPHA, "adapter.lora.alpha" },
src/llama-arch.h
+1
View File
@@ -314,6 +314,7 @@ enum llm_kv {
LLM_KV_TOKENIZER_RWKV, LLM_KV_TOKENIZER_RWKV,
LLM_KV_TOKENIZER_CHAT_TEMPLATE, LLM_KV_TOKENIZER_CHAT_TEMPLATE,
LLM_KV_TOKENIZER_NORMALIZER_LOWERCASE, LLM_KV_TOKENIZER_NORMALIZER_LOWERCASE,
LLM_KV_TOKENIZER_NORMALIZER_STRIP_ACCENTS,
LLM_KV_TOKENIZER_FIM_PRE_ID, LLM_KV_TOKENIZER_FIM_PRE_ID,
LLM_KV_TOKENIZER_FIM_SUF_ID, LLM_KV_TOKENIZER_FIM_SUF_ID,
LLM_KV_TOKENIZER_FIM_MID_ID, LLM_KV_TOKENIZER_FIM_MID_ID,
src/llama-vocab.cpp
+23 -12
View File
@@ -764,7 +764,7 @@ struct llm_tokenizer_wpm_session {
void tokenize(const std::string & text, std::vector<llama_token> & output) { void tokenize(const std::string & text, std::vector<llama_token> & output) {
// normalize and split by whitespace // normalize and split by whitespace
std::vector<std::string> words = preprocess(text, vocab.get_normalizer_lowercase()); std::vector<std::string> words = preprocess(text, vocab.get_normalizer_opts());
// bos token prepended already // bos token prepended already
// find the longest tokens that form the words // find the longest tokens that form the words
@@ -809,11 +809,14 @@ struct llm_tokenizer_wpm_session {
} }
// TODO: reduce string copies by using cpts_offs array // TODO: reduce string copies by using cpts_offs array
static std::vector<std::string> preprocess(const std::string & text, bool lowercase) { static std::vector<std::string> preprocess(const std::string & text, const llama_vocab::normalizer_options & normalizer_opts) {
const std::vector<uint32_t> cpts_nfd = unicode_cpts_normalize_nfd(unicode_cpts_from_utf8(text)); std::vector<uint32_t> cpts = unicode_cpts_from_utf8(text);
if (normalizer_opts.strip_accents) {
cpts = unicode_cpts_normalize_nfd(cpts);
}
std::vector<std::string> words(1, ""); std::vector<std::string> words(1, "");
for (const uint32_t cpt : cpts_nfd) { for (const uint32_t cpt : cpts) {
const auto flags = unicode_cpt_flags_from_cpt(cpt); const auto flags = unicode_cpt_flags_from_cpt(cpt);
if (flags.is_whitespace) { if (flags.is_whitespace) {
@@ -828,7 +831,11 @@ struct llm_tokenizer_wpm_session {
continue; continue;
} }
const std::string s = unicode_cpt_to_utf8(lowercase ? unicode_tolower(cpt) : cpt); if (normalizer_opts.strip_accents && flags.is_accent_mark) {
continue;
}
const std::string s = unicode_cpt_to_utf8(normalizer_opts.lowercase ? unicode_tolower(cpt) : cpt);
if (flags.is_punctuation || ( cpt < 0x7F && flags.is_symbol ) || is_chinese_char(cpt)) { if (flags.is_punctuation || ( cpt < 0x7F && flags.is_symbol ) || is_chinese_char(cpt)) {
if (words.back().size()) { // finish previous word if any if (words.back().size()) { // finish previous word if any
words.emplace_back(); words.emplace_back();
@@ -1692,7 +1699,7 @@ struct llm_tokenizer_whitespace_session : llm_tokenizer_bpe_session {
llm_tokenizer_whitespace_session(const llama_vocab & vocab, const llm_tokenizer_bpe & tokenizer) : llm_tokenizer_bpe_session{vocab, tokenizer}, vocab{vocab} {} llm_tokenizer_whitespace_session(const llama_vocab & vocab, const llm_tokenizer_bpe & tokenizer) : llm_tokenizer_bpe_session{vocab, tokenizer}, vocab{vocab} {}
void tokenize(const std::string & text, std::vector<llama_token> & output) override { void tokenize(const std::string & text, std::vector<llama_token> & output) override {
const bool lowercase = vocab.get_normalizer_lowercase(); const bool lowercase = vocab.get_normalizer_opts().lowercase;
std::string segment; std::string segment;
auto flush = [&]() { auto flush = [&]() {
@@ -1797,7 +1804,9 @@ struct llama_vocab::impl {
bool remove_extra_whitespaces = false; bool remove_extra_whitespaces = false;
bool escape_whitespaces = true; bool escape_whitespaces = true;
bool treat_whitespace_as_suffix = false; bool treat_whitespace_as_suffix = false;
bool normalizer_lowercase = true; // Lowercase normalizer (tokenizer.json)
// BertNormalizer options
llama_vocab::normalizer_options normalizer_opts;
std::unordered_map<std::string, llama_token> token_to_id; std::unordered_map<std::string, llama_token> token_to_id;
std::vector<token_data> id_to_token; std::vector<token_data> id_to_token;
@@ -2172,7 +2181,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
} else if ( } else if (
tokenizer_pre == "whitespace") { tokenizer_pre == "whitespace") {
pre_type = LLAMA_VOCAB_PRE_TYPE_WHITESPACE; pre_type = LLAMA_VOCAB_PRE_TYPE_WHITESPACE;
normalizer_lowercase = false; normalizer_opts.lowercase = false;
} else if ( } else if (
tokenizer_pre == "refact") { tokenizer_pre == "refact") {
pre_type = LLAMA_VOCAB_PRE_TYPE_REFACT; pre_type = LLAMA_VOCAB_PRE_TYPE_REFACT;
@@ -2532,8 +2541,10 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
} }
} }
// Lowercase normalizer flag (consulted by WPM / whitespace BPE) // BertNormalizer options
ml.get_key(LLM_KV_TOKENIZER_NORMALIZER_LOWERCASE, normalizer_lowercase, false); ml.get_key(LLM_KV_TOKENIZER_NORMALIZER_LOWERCASE, normalizer_opts.lowercase, false);
normalizer_opts.strip_accents = normalizer_opts.lowercase;
ml.get_key(LLM_KV_TOKENIZER_NORMALIZER_STRIP_ACCENTS, normalizer_opts.strip_accents, false);
// suppress tokens // suppress tokens
{ {
@@ -3969,8 +3980,8 @@ bool llama_vocab::get_treat_whitespace_as_suffix() const {
return pimpl->treat_whitespace_as_suffix; return pimpl->treat_whitespace_as_suffix;
} }
bool llama_vocab::get_normalizer_lowercase() const { const llama_vocab::normalizer_options & llama_vocab::get_normalizer_opts() const {
return pimpl->normalizer_lowercase; return pimpl->normalizer_opts;
} }
const std::vector<llama_token> & llama_vocab::get_suppress_tokens() const { const std::vector<llama_token> & llama_vocab::get_suppress_tokens() const {
src/llama-vocab.h
+7 -1
View File
@@ -76,6 +76,12 @@ struct llama_vocab {
llama_token_attr attr; llama_token_attr attr;
}; };
struct normalizer_options {
bool lowercase = true;
bool strip_accents = true;
// TODO: clean_text, handle_chinese_chars
};
llama_vocab(); llama_vocab();
~llama_vocab(); ~llama_vocab();
@@ -141,7 +147,7 @@ struct llama_vocab {
bool get_remove_extra_whitespaces () const; bool get_remove_extra_whitespaces () const;
bool get_escape_whitespaces () const; bool get_escape_whitespaces () const;
bool get_treat_whitespace_as_suffix() const; bool get_treat_whitespace_as_suffix() const;
bool get_normalizer_lowercase () const; const normalizer_options & get_normalizer_opts() const;
const std::vector<llama_token> & get_suppress_tokens() const; const std::vector<llama_token> & get_suppress_tokens() const;