huggingface/candle
1
0
Fork 0
mirror of https://github.com/huggingface/candle.git synced 2025-06-20 20:09:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Optimizing decode matmul (Phi at 28tok/s on M3).

Browse Source 
Adding some benchmark in order to help checking out matmul performance.
This commit is contained in:
Nicolas Patry
2023-12-20 09:54:19 +01:00
parent 03641293ee
commit 9b5e4843a6
4 changed files with 66 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
candle-metal-kernels/src/lib.rs
View File

@ -1297,11 +1297,21 @@ pub fn call_gemm(
let batched = b > 1;
let fused_activation = false;
let fused_bias = false;
let m_simd = 16;
let n_simd = 16;
let k_simd = 16;
let m_splits = 2;
let n_splits = 2;
let (m_simd, n_simd, k_simd, m_splits, n_splits) = if m == 1 {
let m_simd = 16;
let n_simd = 8;
let k_simd = 64;
let m_splits = 1;
let n_splits = 1;
(m_simd, n_simd, k_simd, m_splits, n_splits)
} else {
let m_simd = 40;
let n_simd = 40;
let k_simd = 8;
let m_splits = 1;
let n_splits = 1;
(m_simd, n_simd, k_simd, m_splits, n_splits)
};
let constants = Some(ConstantValues::new(vec![
(0, Value::USize(m)),
(1, Value::USize(n)),