huggingface/candle
1
0
Fork 0
mirror of https://github.com/huggingface/candle.git synced 2025-06-16 18:48:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

Properly handle the batch dimension in cuda quantized matmul. (#1832)

Browse Source
This commit is contained in:
Laurent Mazare
2024-03-10 20:23:43 +01:00
committed by GitHub
parent 0c5eecbc0f
commit df5f69444e
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
candle-core/src/quantized/cuda.rs
View File

@ -313,7 +313,7 @@ impl QCudaStorage {
} }
let data_f32 = self.dequantize(n * k)?; let data_f32 = self.dequantize(n * k)?;
let rhs_l = crate::Layout::new((k, n).into(), vec![1, k], 0); let rhs_l = crate::Layout::new((k, n).into(), vec![1, k], 0).broadcast_as((b, k, n))?;
let out = storage.matmul(&data_f32, (b, m, n, k), layout, &rhs_l)?; let out = storage.matmul(&data_f32, (b, m, n, k), layout, &rhs_l)?;
let mut out_shape = layout.shape().dims().to_vec(); let mut out_shape = layout.shape().dims().to_vec();
out_shape.pop(); out_shape.pop();