huggingface/candle
1
0
Fork 0
mirror of https://github.com/huggingface/candle.git synced 2025-06-18 19:47:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Non random for better quantization quality

Browse Source
This commit is contained in:
Nicolas Patry
2024-01-06 15:16:01 +01:00
parent 504d0b9ac7
commit 2bbd544832
1 changed files with 27 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
candle-core/tests/quantized_tests.rs
View File

@ -553,10 +553,10 @@ fn get_random_tensors(
let mut rng = StdRng::seed_from_u64(314159265358979); let mut rng = StdRng::seed_from_u64(314159265358979);
let lhs = (0..m * k) let lhs = (0..m * k)
.map(|_| rng.gen::<f32>() - 0.5) .map(|i| i as f32 / (m * k) as f32)
.collect::<Vec<_>>(); .collect::<Vec<_>>();
let rhs = (0..n * k) let rhs = (0..n * k)
.map(|_| rng.gen::<f32>() - 0.5) .map(|i| i as f32 / (n * k) as f32)
.collect::<Vec<_>>(); .collect::<Vec<_>>();
let lhs = Tensor::from_vec(lhs, (m, k), device)?; let lhs = Tensor::from_vec(lhs, (m, k), device)?;
@ -599,20 +599,30 @@ fn quantized_matmul_q3k() -> Result<()> {
let cpu = &Device::Cpu; let cpu = &Device::Cpu;
let (m, k, n) = (11, 512, 21); let (m, k, n) = (11, 512, 21);
let (lhs, rhs, mm) = get_random_tensors(m, k, n, cpu)?; let (lhs, rhs, mm) = get_random_tensors(m, k, n, cpu)?;
assert_eq!(mm.dims(), [m, n]); // assert_eq!(mm.dims(), [m, n]);
let dst = mm.flatten_all()?.to_vec1::<f32>()?; // let dst = mm.flatten_all()?.to_vec1::<f32>()?;
let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); // let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]);
assert_eq!(dst, [1.262, 1.513, -0.208, 1.702]); // assert_eq!(dst, [1.262, 1.513, -0.208, 1.702]);
let rhs = quantized::QTensor::quantize::<BlockQ3K>(&rhs)?; let rhs = quantized::QTensor::quantize::<BlockQ3K>(&rhs)?;
let rhs = quantized::QMatMul::from_qtensor(rhs)?; let rhs = quantized::QMatMul::from_qtensor(rhs)?;
let mm = rhs.forward(&lhs)?; let qmm = rhs.forward(&lhs)?;
assert_eq!(mm.dims(), [m, n]); let error: f32 = ((&mm - &qmm)?.abs()? / &mm.abs()?)?
let dst = mm.flatten_all()?.to_vec1::<f32>()?; .sum_all()?
.to_scalar()?;
let error = error / (m * n) as f32;
assert_eq!(qmm.dims(), [m, n]);
let dst = qmm.flatten_all()?.to_vec1::<f32>()?;
let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]);
assert_eq!(dst, [1.029, 1.418, -0.314, 1.495]); assert_eq!(dst, [1.029, 1.418, -0.314, 1.495]);
assert!(
error < 0.01,
"{error} is too big, shouldn't exceed a few percent. \nGot:{qmm}\nExpected:\n{mm} "
);
ggml_matmul_error_test::<BlockQ3K>()?; ggml_matmul_error_test::<BlockQ3K>()?;
Ok(()) Ok(())
@ -625,10 +635,10 @@ fn quantized_matmul_q4k() -> Result<()> {
let cpu = &Device::Cpu; let cpu = &Device::Cpu;
let (m, k, n) = (11, 512, 21); let (m, k, n) = (11, 512, 21);
let (lhs, rhs, mm) = get_random_tensors(m, k, n, cpu)?; let (lhs, rhs, mm) = get_random_tensors(m, k, n, cpu)?;
assert_eq!(mm.dims(), [m, n]); // assert_eq!(mm.dims(), [m, n]);
let dst = mm.flatten_all()?.to_vec1::<f32>()?; // let dst = mm.flatten_all()?.to_vec1::<f32>()?;
let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); // let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]);
assert_eq!(dst, [1.262, 1.513, -0.208, 1.702]); // assert_eq!(dst, [1.262, 1.513, -0.208, 1.702]);
let rhs = quantized::QTensor::quantize::<BlockQ4K>(&rhs)?; let rhs = quantized::QTensor::quantize::<BlockQ4K>(&rhs)?;
let rhs = quantized::QMatMul::from_qtensor(rhs)?; let rhs = quantized::QMatMul::from_qtensor(rhs)?;
@ -644,10 +654,10 @@ fn quantized_matmul_q4k() -> Result<()> {
"{error} is too big, shouldn't exceed a few percent. \nGot:{qmm}\nExpected:\n{mm} " "{error} is too big, shouldn't exceed a few percent. \nGot:{qmm}\nExpected:\n{mm} "
); );
assert_eq!(mm.dims(), [m, n]); // assert_eq!(mm.dims(), [m, n]);
let dst = mm.flatten_all()?.to_vec1::<f32>()?; // let dst = mm.flatten_all()?.to_vec1::<f32>()?;
let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); // let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]);
assert_eq!(dst, [1.125, 1.435, -0.201, 1.589]); // assert_eq!(dst, [1.125, 1.435, -0.201, 1.589]);
ggml_matmul_error_test::<BlockQ4K>()?; ggml_matmul_error_test::<BlockQ4K>()?;