Add the StarCoder2 model. (#1779)

* Add the StarCoder2 model.

* Add the example code and get things to work.

* And also tweak the readme.
This commit is contained in:
Laurent Mazare
2024-02-28 21:02:41 +01:00
committed by GitHub
parent 57267cd536
commit 4fd00b8900
6 changed files with 609 additions and 3 deletions

View File

@ -40,7 +40,7 @@ impl TokenOutputStream {
};
self.tokens.push(token);
let text = self.decode(&self.tokens[self.prev_index..])?;
if text.len() > prev_text.len() && text.chars().last().unwrap().is_alphabetic() {
if text.len() > prev_text.len() && text.chars().last().unwrap().is_alphanumeric() {
let text = text.split_at(prev_text.len());
self.prev_index = self.current_index;
self.current_index = self.tokens.len();