vocab : refactor normalizer flags into options struct, add strip_accents (#24371)

* vocab : refactor normalizer flags into options struct, add strip_accents

* Update src/llama-vocab.h

Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@scala.com>

* Update src/llama-vocab.cpp

Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@scala.com>

---------

Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@scala.com>

gguf-py/gguf/constants.py

@@ -272,7 +272,8 @@ class Keys:
         CHAT_TEMPLATE_N      = "tokenizer.chat_template.{name}"
         CHAT_TEMPLATES       = "tokenizer.chat_templates"
         # Normalizer constants
-        NORMALIZER_LOWERCASE = "tokenizer.ggml.normalizer.lowercase"
+        NORMALIZER_LOWERCASE     = "tokenizer.ggml.normalizer.lowercase"
+        NORMALIZER_STRIP_ACCENTS = "tokenizer.ggml.normalizer.strip_accents"
         # FIM/Infill special tokens constants
         FIM_PRE_ID           = "tokenizer.ggml.fim_pre_token_id"
         FIM_SUF_ID           = "tokenizer.ggml.fim_suf_token_id"

gguf-py/gguf/gguf_writer.py

@@ -1124,6 +1124,9 @@ class GGUFWriter:
     def add_normalizer_lowercase(self, value: bool) -> None:
         self.add_bool(Keys.Tokenizer.NORMALIZER_LOWERCASE, value)
 
+    def add_normalizer_strip_accents(self, value: bool) -> None:
+        self.add_bool(Keys.Tokenizer.NORMALIZER_STRIP_ACCENTS, value)
+
     def add_eot_token_id(self, id: int) -> None:
         self.add_uint32(Keys.Tokenizer.EOT_ID, id)
 

gguf-py/gguf/vocab.py

@@ -53,6 +53,7 @@ class SpecialVocab:
     special_token_ids: dict[str, int]
     chat_template: str | Sequence[Mapping[str, str]] | None
     normalizer_lowercase: bool | None
+    normalizer_strip_accents: bool | None
 
     def __init__(
         self, path: str | os.PathLike[str], load_merges: bool = False,
@@ -66,6 +67,7 @@ class SpecialVocab:
         self.merges = []
         self.chat_template = None
         self.normalizer_lowercase = None
+        self.normalizer_strip_accents = None
         if special_token_types is not None:
             self.special_token_types = special_token_types
         else:
@@ -108,6 +110,10 @@ class SpecialVocab:
             if not quiet:
                 logger.info(f'Setting normalizer_lowercase to {self.normalizer_lowercase}')
             gw.add_normalizer_lowercase(self.normalizer_lowercase)
+        if self.normalizer_strip_accents is not None:
+            if not quiet:
+                logger.info(f'Setting normalizer_strip_accents to {self.normalizer_strip_accents}')
+            gw.add_normalizer_strip_accents(self.normalizer_strip_accents)
 
     def _load(self, path: Path) -> None:
         self._try_load_from_tokenizer_json(path)
@@ -155,17 +161,21 @@ class SpecialVocab:
     def _parse_normalizer(self, normalizer: dict) -> None:
         # ref: https://huggingface.co/docs/tokenizers/api/normalizers
         #
-        # Detects lowercase normalization in three possible formats:
-        # 1. Standalone: {"type": "Lowercase"}
-        # 2. BertNormalizer attribute: {"type": "BertNormalizer", "lowercase": true, ...}
-        # 3. Nested in Sequence: {"type": "Sequence", "normalizers": [...]}
+        # Extracts normalizer flags from three possible formats:
+        # 1. Standalone:           {"type": "Lowercase"}
+        # 2. BertNormalizer attrs: {"type": "BertNormalizer", ...}
+        # 3. Nested in Sequence:   {"type": "Sequence", "normalizers": [...]}
 
         normalizer_type = normalizer.get('type')
         if normalizer_type == 'Lowercase':
             self.normalizer_lowercase = True
+        elif normalizer_type == 'StripAccents':
+            self.normalizer_strip_accents = True
         elif normalizer_type == 'BertNormalizer':
             if 'lowercase' in normalizer:
                 self.normalizer_lowercase = normalizer['lowercase']
+            if 'strip_accents' in normalizer:
+                self.normalizer_strip_accents = normalizer['strip_accents']
         elif normalizer_type == 'Sequence':
             for norm in normalizer.get('normalizers', []):
                 self._parse_normalizer(norm)