Avoid a contiguous call in the quantized phi 3 model. (#2209)

* Simplify the KvCache api.

* Avoid a contiguous call in the quantized phi3 model.
This commit is contained in:
Laurent Mazare
2024-05-23 21:24:55 +02:00
committed by GitHub
parent 45e235a747
commit d54e02d73d

View File

@ -146,7 +146,7 @@ impl LayerWeights {
};
let att = candle_nn::ops::softmax_last_dim(&att)?;
// Convert to contiguous as matmul doesn't support strided vs for now.
att.matmul(&v.contiguous()?)?
att.matmul(&v)?
};
let y = y.transpose(1, 2)?.reshape(&[b_sz, seq_len, n_embd])?;
let y = self.attn_output.forward(&y)?;