huggingface/candle
1
0
Fork 0
mirror of https://github.com/huggingface/candle.git synced 2025-06-22 12:28:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

More kv-cache testing.

Browse Source
This commit is contained in:
Laurent
2024-09-22 12:51:07 +02:00
parent f9579f80be
commit 6547c4bfc3
2 changed files with 27 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
candle-nn/src/kv_cache.rs
View File

@ -228,7 +228,7 @@ impl RotatingCache {
self.offset = 0;
} else {
let rem_len = self.max_seq_len - self.offset;
if rem_len <= seq_len {
if seq_len <= rem_len {
ad.slice_set(src, self.dim, self.offset)?;
self.offset = (self.offset + seq_len) % self.max_seq_len;
} else {