huggingface/candle
1
0
Fork 0
mirror of https://github.com/huggingface/candle.git synced 2025-06-22 12:28:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add a KV cache to T5. (#873)

Browse Source 
* Add a KV cache to T5.

* Suggest using release mode.

* Use the kv cache in decoding.

* Add a comment.
This commit is contained in:
Laurent Mazare
2023-09-17 09:00:45 +02:00
committed by GitHub
parent 8658df3485
commit 1a276b5da7
5 changed files with 577 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
candle-examples/examples/musicgen/main.rs
View File

@ -77,7 +77,7 @@ fn main() -> Result<()> {
let model = model.deserialize()?;
let vb = VarBuilder::from_safetensors(vec![model], DTYPE, &device);
let config = GenConfig::small();
let model = MusicgenForConditionalGeneration::load(vb, config)?;
let mut model = MusicgenForConditionalGeneration::load(vb, config)?;
let tokens = tokenizer
.encode(args.prompt.as_str(), true)