huggingface/candle
1
0
Fork 0
mirror of https://github.com/huggingface/candle.git synced 2025-06-16 18:48:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

Ignore rotary for mistral.

Browse Source
This commit is contained in:
Nicolas Patry
2024-01-05 21:55:13 +01:00
parent 12fc4af8f2
commit 9cd0cc1f65
1 changed files with 7 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
candle-transformers/src/models/quantized_mistral.rs
View File

@ -165,9 +165,13 @@ impl Attention {
.reshape((b_sz, q_len, self.num_kv_heads, self.head_dim))?
.transpose(1, 2)?;
let (query_states, key_states) =
self.rotary_emb
.apply_rotary_emb_qkv(&query_states, &key_states, seqlen_offset)?;
// let (query_states1, key_states1) =
// self.rotary_emb
// .apply_rotary_emb_qkv(&query_states, &key_states, seqlen_offset)?;
// println!("{query_states:?} {query_states1:?}");
// println!("{key_states:?} {key_states1:?}");
let query_states = query_states.contiguous()?;
let key_states = key_states.contiguous()?;
let (key_states, value_states) = match &self.kv_cache {
None => (key_states, value_states),