From 74497e6bf738366d4c599b42826e204fbcb96f37 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Sun, 11 Feb 2024 08:52:36 +0100 Subject: [PATCH] Fixing the qwen tokenizer location. (#1693) Using the chatglm one causes a bug where the "<|endoftext|>" is not found. --- candle-examples/examples/qwen/main.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/candle-examples/examples/qwen/main.rs b/candle-examples/examples/qwen/main.rs index add0d519..d040d4b0 100644 --- a/candle-examples/examples/qwen/main.rs +++ b/candle-examples/examples/qwen/main.rs @@ -235,9 +235,7 @@ fn main() -> Result<()> { )); let tokenizer_filename = match args.tokenizer_file { Some(file) => std::path::PathBuf::from(file), - None => api - .model("lmz/candle-chatglm".to_string()) - .get("chatglm-tokenizer.json")?, + None => repo.get("tokenizer.json")?, }; let filenames = match args.weight_files { Some(files) => files