huggingface/candle
1
0
Fork 0
mirror of https://github.com/huggingface/candle.git synced 2025-06-16 10:38:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Tensor -> QTensor conversion (#496)

Browse Source 
* Sketch some qmatmul test.

* Add the quantization function.

* More testing.

* Make the test smaller and faster.

* Add some shape checking.
This commit is contained in:
Laurent Mazare
2023-08-18 08:19:20 +01:00
committed by GitHub
parent 26fd37b348
commit a22b1bed7b
4 changed files with 87 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
candle-core/examples/cpu_benchmarks.rs
View File

@ -90,7 +90,7 @@ impl Benchmark for QMatMul {
type RunResult = Tensor;
fn preprocess() -> Result<Self::PreProcessData> {
let zeros = vec![candle_core::quantized::k_quants::BlockQ4_0::zeros(); 4096 * 11008 / 32];
let mm = candle_core::quantized::QTensor::new(zeros, (4096, 11008));
let mm = candle_core::quantized::QTensor::new(zeros, (4096, 11008))?;
let mm = candle_core::quantized::QMatMul::from_qtensor(mm);
let arg = Tensor::randn(0f32, 1., (128, 11008), &Device::Cpu)?;
Ok((mm, arg))