nikola/llama_cpp
1
0
Fork 0
Code Issues Pull Requests Actions 23 Packages Projects Releases Wiki Activity

vocab : refactor normalizer flags into options struct, add strip_accents (#24371)

Browse Source 
* vocab : refactor normalizer flags into options struct, add strip_accents

* Update src/llama-vocab.h

Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@scala.com>

* Update src/llama-vocab.cpp

Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@scala.com>

---------

Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@scala.com>
This commit is contained in:
o7si
2026-06-11 10:36:50 +03:00
committed by GitHub
co-authored by Sigbjørn Skjæret
parent db94854ff5
commit 68f30663cf
7 changed files with 84 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gguf-py/gguf/vocab.py
+14 -4
View File
@@ -53,6 +53,7 @@ class SpecialVocab:
special_token_ids: dict[str, int]
chat_template: str | Sequence[Mapping[str, str]] | None
normalizer_lowercase: bool | None
normalizer_strip_accents: bool | None
def __init__(
self, path: str | os.PathLike[str], load_merges: bool = False,
@@ -66,6 +67,7 @@ class SpecialVocab:
self.merges = []
self.chat_template = None
self.normalizer_lowercase = None
self.normalizer_strip_accents = None
if special_token_types is not None:
self.special_token_types = special_token_types
else:
@@ -108,6 +110,10 @@ class SpecialVocab:
if not quiet:
logger.info(f'Setting normalizer_lowercase to {self.normalizer_lowercase}')
gw.add_normalizer_lowercase(self.normalizer_lowercase)
if self.normalizer_strip_accents is not None:
if not quiet:
logger.info(f'Setting normalizer_strip_accents to {self.normalizer_strip_accents}')
gw.add_normalizer_strip_accents(self.normalizer_strip_accents)
def _load(self, path: Path) -> None:
self._try_load_from_tokenizer_json(path)
@@ -155,17 +161,21 @@ class SpecialVocab:
def _parse_normalizer(self, normalizer: dict) -> None:
# ref: https://huggingface.co/docs/tokenizers/api/normalizers
#
# Detects lowercase normalization in three possible formats:
# 1. Standalone: {"type": "Lowercase"}
# 2. BertNormalizer attribute: {"type": "BertNormalizer", "lowercase": true, ...}
# 3. Nested in Sequence: {"type": "Sequence", "normalizers": [...]}
# Extracts normalizer flags from three possible formats:
# 1. Standalone: {"type": "Lowercase"}
# 2. BertNormalizer attrs: {"type": "BertNormalizer", ...}
# 3. Nested in Sequence: {"type": "Sequence", "normalizers": [...]}
normalizer_type = normalizer.get('type')
if normalizer_type == 'Lowercase':
self.normalizer_lowercase = True
elif normalizer_type == 'StripAccents':
self.normalizer_strip_accents = True
elif normalizer_type == 'BertNormalizer':
if 'lowercase' in normalizer:
self.normalizer_lowercase = normalizer['lowercase']
if 'strip_accents' in normalizer:
self.normalizer_strip_accents = normalizer['strip_accents']
elif normalizer_type == 'Sequence':
for norm in normalizer.get('normalizers', []):
self._parse_normalizer(norm)