huggingface/candle
1
0
Fork 0
mirror of https://github.com/huggingface/candle.git synced 2025-06-19 11:56:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Generic implementation of vecdot for q80. (#596)

Browse Source 
* Generic implementation of vecdot for q80.

* Add support for code-llama 7b.

* Support more code-llama.
This commit is contained in:
Laurent Mazare
2023-08-25 09:04:05 +01:00
committed by GitHub
parent d8ba0452dc
commit c093b03d51
2 changed files with 41 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
candle-core/src/quantized/k_quants.rs
View File

@ -421,8 +421,24 @@ impl GgmlType for BlockQ8_0 {
Ok(())
}
fn vec_dot(_: usize, _: &[Self], _: &[Self::VecDotType]) -> Result<f32> {
todo!()
fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result<f32> {
let qk = QK8_0;
if n % QK8_0 != 0 {
crate::bail!("vec_dot_q8_0_q8_0: {n} is not divisible by {qk}")
}
// Generic implementation.
let mut sumf = 0f32;
for (xs, ys) in xs.iter().zip(ys.iter()) {
let sum_i = xs
.qs
.iter()
.zip(ys.qs.iter())
.map(|(&x, &y)| x as i32 * y as i32)
.sum::<i32>();
sumf += sum_i as f32 * f16::to_f32(xs.d) * f16::to_f32(ys.d)
}
Ok(sumf)
}
}