mirror of
https://github.com/huggingface/candle.git
synced 2025-06-16 18:48:51 +00:00
2f22afd80e
* Boilerplate for the quantized cuda support. * More basic cuda support. * More cuda quantization (quantize on cpu for now). * Add the dequantization bit. * Start adding some dedicated cuda kernels from llama.cpp. * Move the kernel code. * Start interfacing with the kernel. * Tweak the kernel launch params. * Bugfix for quantized metal. * Fix some clippy lints. * Tweak the launch parameters. * Tweak cuda basics to perform a quantized matmul. * Perform the dequantization on the cpu + use cublas for matmul. * Add the dequantization kernel. * Test the qmatmul. * More kernels. * Matmul-vec kernel. * Add a couple kernels. * More dequantization kernels.
37 lines
1.1 KiB
Rust
37 lines
1.1 KiB
Rust
#[cfg(feature = "accelerate")]
|
|
extern crate accelerate_src;
|
|
|
|
#[cfg(feature = "mkl")]
|
|
extern crate intel_mkl_src;
|
|
|
|
use anyhow::Result;
|
|
use candle_core::{Device, Module, Tensor};
|
|
|
|
use candle_core::quantized::{QMatMul, QTensor};
|
|
|
|
fn main() -> Result<()> {
|
|
let device = Device::new_cuda(0)?;
|
|
let q = Tensor::randn(0f32, 1.0, (72, 32), &device)?;
|
|
let q_cpu = q.to_device(&Device::Cpu)?;
|
|
let q = QTensor::quantize(&q, candle_core::quantized::GgmlDType::Q4_0)?;
|
|
let q = QMatMul::from_qtensor(q)?;
|
|
let x = Tensor::randn(0f32, 1.0, (5, 32), &device)?;
|
|
let res_q_cuda = q.forward(&x)?;
|
|
println!("{res_q_cuda}");
|
|
|
|
let q_cpu = QTensor::quantize(&q_cpu, candle_core::quantized::GgmlDType::Q4_0)?;
|
|
let q_cpu_tensor = q_cpu.dequantize(&Device::Cpu)?;
|
|
let q_cpu = QMatMul::from_qtensor(q_cpu)?;
|
|
let x_cpu = x.to_device(&Device::Cpu)?;
|
|
let res_q_cpu = q_cpu.forward(&x_cpu)?;
|
|
println!("{res_q_cpu}");
|
|
|
|
let res_mm = x_cpu.matmul(&q_cpu_tensor.t()?)?;
|
|
let diff = (res_mm - res_q_cuda.to_device(&Device::Cpu))?
|
|
.abs()?
|
|
.flatten_all()?
|
|
.max(0)?;
|
|
println!("{diff}");
|
|
Ok(())
|
|
}
|