huggingface/candle
1
0
Fork 0
mirror of https://github.com/huggingface/candle.git synced 2025-06-16 10:38:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Remove the end of text tokens. (#289)

Browse Source
This commit is contained in:
Laurent Mazare
2023-07-31 20:43:57 +01:00
committed by GitHub
parent 9ae1f6afee
commit 6b98b66eb3
2 changed files with 61 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
candle-examples/examples/llama2-c/main.rs
View File

@ -266,7 +266,8 @@ fn run_eval(tokenizer: Tokenizer, config_path: &std::path::PathBuf, args: Args)
let file = std::io::BufReader::new(file);
let mut tokens = vec![];
for line in file.lines() {
let line = tokenizer.encode(line?, false).map_err(E::msg)?;
let line = line?.replace("<|endoftext|>", "");
let line = tokenizer.encode(line, false).map_err(E::msg)?;
tokens.push(line.get_ids().to_vec())
}
let tokens = tokens.concat();