nikola/llama_cpp
1
0
Fork 0
Code Issues Pull Requests Actions 25 Packages Projects Releases Wiki Activity

Missing tokenizer.model error during gguf conversion (#6443)

Browse Source 
Co-authored-by: Jared Van Bortel <jared@nomic.ai>
This commit is contained in:
Abhishek Gopinath K
2024-04-03 11:42:52 -04:00
committed by GitHub
co-authored by Jared Van Bortel
parent 1ff4d9f3d6
commit db214fa578
1 changed files with 1 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
convert-hf-to-gguf.py
+1 -2
View File
@@ -323,8 +323,7 @@ class Model(ABC):
toktypes: list[int] = [] toktypes: list[int] = []
if not tokenizer_path.is_file(): if not tokenizer_path.is_file():
print(f'Error: Missing {tokenizer_path}', file=sys.stderr) raise FileNotFoundError(f"File not found: {tokenizer_path}")
sys.exit(1)
tokenizer = SentencePieceProcessor(str(tokenizer_path)) tokenizer = SentencePieceProcessor(str(tokenizer_path))
vocab_size = self.hparams.get('vocab_size', tokenizer.vocab_size()) vocab_size = self.hparams.get('vocab_size', tokenizer.vocab_size())