Fixing the qwen tokenizer location. (#1693)

Using the chatglm one causes a bug where the "<|endoftext|>" is not
found.
This commit is contained in:
Nicolas Patry
2024-02-11 08:52:36 +01:00
committed by GitHub
parent 8ab384e63d
commit 74497e6bf7

View File

@ -235,9 +235,7 @@ fn main() -> Result<()> {
));
let tokenizer_filename = match args.tokenizer_file {
Some(file) => std::path::PathBuf::from(file),
None => api
.model("lmz/candle-chatglm".to_string())
.get("chatglm-tokenizer.json")?,
None => repo.get("tokenizer.json")?,
};
let filenames = match args.weight_files {
Some(files) => files