huggingface/candle
1
0
Fork 0
mirror of https://github.com/huggingface/candle.git synced 2025-06-15 10:26:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

ChatGLM custom tokenizer. (#1687)

Browse Source
This commit is contained in:
Laurent Mazare
2024-02-10 10:47:04 +01:00
committed by GitHub
parent 90447bc993
commit 1c8d61f051
1 changed files with 3 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
candle-examples/examples/qwen/main.rs
View File

@ -235,7 +235,9 @@ fn main() -> Result<()> {
)); ));
let tokenizer_filename = match args.tokenizer_file { let tokenizer_filename = match args.tokenizer_file {
Some(file) => std::path::PathBuf::from(file), Some(file) => std::path::PathBuf::from(file),
None => repo.get("tokenizer.json")?, None => api
.model("lmz/candle-chatglm".to_string())
.get("chatglm-tokenizer.json")?,
}; };
let filenames = match args.weight_files { let filenames = match args.weight_files {
Some(files) => files Some(files) => files