From c5229087a5150571843e8aa7b755180e640a0710 Mon Sep 17 00:00:00 2001 From: ynankani Date: Thu, 28 May 2026 07:16:17 +0000 Subject: [PATCH] convert : add FP8 to Q8 conversion (#23250) Signed-off-by: ynankani --- conversion/base.py | 35 +++++++++++++++++++++++++++++------ convert_hf_to_gguf.py | 7 ++++++- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/conversion/base.py b/conversion/base.py index e1c15a958..9cddd1340 100644 --- a/conversion/base.py +++ b/conversion/base.py @@ -119,7 +119,8 @@ class ModelBase: small_first_shard: bool = False, hparams: dict[str, Any] | None = None, remote_hf_model_id: str | None = None, disable_mistral_community_chat_template: bool = False, sentence_transformers_dense_modules: bool = False, - fuse_gate_up_exps: bool = False): + fuse_gate_up_exps: bool = False, + fp8_as_q8: bool = False): if type(self) is ModelBase or \ type(self) is TextModel or \ type(self) is MmprojModel: @@ -148,6 +149,8 @@ class ModelBase: self.dir_model_card = dir_model # overridden in convert_lora_to_gguf.py self._is_nvfp4 = False self._is_mxfp4 = False + self._fp8_as_q8 = fp8_as_q8 + self._fp8_dequantized: set[str] = set() # Apply heuristics to figure out typical tensor encoding based on first tensor's dtype # NOTE: can't use field "torch_dtype" in config.json, because some finetunes lie. @@ -429,6 +432,8 @@ class ModelBase: s = self.model_tensors[name] self.model_tensors[weight_name] = lambda w=w, s=s, bs=block_size: dequant_simple(w(), s(), bs) tensors_to_remove.append(name) + if self._fp8_as_q8: + self._fp8_dequantized.add(weight_name) if name.endswith(".activation_scale"): # unused tensors_to_remove.append(name) if name.endswith("_activation_scale"): # Mistral-Small-4-119B-2602, unused @@ -440,6 +445,8 @@ class ModelBase: s = self.model_tensors[name] self.model_tensors[weight_name] = lambda w=w, s=s, bs=block_size: dequant_simple(w(), s(), bs) tensors_to_remove.append(name) + if self._fp8_as_q8: + self._fp8_dequantized.add(weight_name) if name.endswith(".qscale_act"): tensors_to_remove.append(name) elif quant_method == "gptq": @@ -483,6 +490,11 @@ class ModelBase: strategy = weight_config.get("strategy") assert strategy == "channel" or strategy == "block" assert weight_config.get("group_size") is None # didn't find a model using this yet + is_fp8 = ( + quant_format == "float-quantized" + and weight_config.get("type") == "float" + and weight_config.get("num_bits") == 8 + ) for name in self.model_tensors.keys(): if name.endswith(".weight_scale"): weight_name = name.removesuffix("_scale") @@ -490,6 +502,8 @@ class ModelBase: s = self.model_tensors[name] self.model_tensors[weight_name] = lambda w=w, s=s: dequant_simple(w(), s(), block_size) tensors_to_remove.append(name) + if self._fp8_as_q8 and is_fp8: + self._fp8_dequantized.add(weight_name) elif quant_format == "pack-quantized": assert weight_config.get("strategy") == "group" assert weight_config.get("type", "int") == "int" @@ -524,10 +538,18 @@ class ModelBase: for name in self.model_tensors.keys(): if name.endswith(".weight_scale"): weight_name = name.removesuffix("_scale") + if weight_name not in self.model_tensors: + tensors_to_remove.append(name) + continue w = self.model_tensors[weight_name] s = self.model_tensors[name] + is_fp8_weight = False + if self._fp8_as_q8: + is_fp8_weight = w().dtype in (torch.float8_e4m3fn, torch.float8_e5m2) self.model_tensors[weight_name] = lambda w=w, s=s: dequant_simple(w(), s(), None) tensors_to_remove.append(name) + if is_fp8_weight: + self._fp8_dequantized.add(weight_name) if name.endswith((".input_scale", ".k_scale", ".v_scale")): tensors_to_remove.append(name) elif quant_method is not None: @@ -615,8 +637,10 @@ class ModelBase: return [(new_name, data_torch)] def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool: - del name, new_name, bid, n_dims # unused - + del new_name, bid # unused + # Force FP8-original tensors to Q8_0 when requested; Q8_0 is faster than F16/BF16. + if self._fp8_as_q8 and name in self._fp8_dequantized and n_dims >= 2: + return gguf.GGMLQuantizationType.Q8_0 return False # some models need extra generated tensors (like rope_freqs) @@ -791,7 +815,7 @@ class ModelBase: if quant_algo != "NVFP4": if nvfp4_compressed_tensors: quant_algo = "NVFP4" - elif any(v.get("quant_algo") == "NVFP4" for v in quant_layers.values() if isinstance(v, dict)): + elif any(str(v.get("quant_algo")).endswith("NVFP4") for v in quant_layers.values() if isinstance(v, dict)): quant_algo = "NVFP4" self._is_nvfp4 = quant_algo == "NVFP4" @@ -2417,10 +2441,9 @@ class MmprojModel(ModelBase): raise KeyError(f"could not find any of: {keys}") def tensor_force_quant(self, name, new_name, bid, n_dims): - del bid, name, n_dims # unused if ".patch_embd.weight" in new_name or ".patch_merger.weight" in new_name: return gguf.GGMLQuantizationType.F16 if self.ftype == gguf.LlamaFileType.MOSTLY_F16 else gguf.GGMLQuantizationType.F32 - return False + return super().tensor_force_quant(name, new_name, bid, n_dims) class LazyTorchTensor(gguf.LazyBase): diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 1d18a1bf9..855275535 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -148,6 +148,10 @@ def parse_args() -> argparse.Namespace: "--fuse-gate-up-exps", action="store_true", help="Fuse gate_exps and up_exps tensors into a single gate_up_exps tensor for MoE models.", ) + parser.add_argument( + "--fp8-as-q8", action="store_true", + help="Store tensors dequantized from FP8 as Q8_0 instead of BF16/F16.", + ) args = parser.parse_args() if not args.print_supported_models and args.model is None: @@ -264,7 +268,8 @@ def main() -> None: small_first_shard=args.no_tensor_first_split, remote_hf_model_id=hf_repo_id, disable_mistral_community_chat_template=disable_mistral_community_chat_template, sentence_transformers_dense_modules=args.sentence_transformers_dense_modules, - fuse_gate_up_exps=args.fuse_gate_up_exps + fuse_gate_up_exps=args.fuse_gate_up_exps, + fp8_as_q8=args.fp8_as_q8, ) if args.vocab_only: