huggingface/candle
1
0
Fork 0
mirror of https://github.com/huggingface/candle.git synced 2025-06-15 02:16:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixed Quantized Qwen3 Model (#2951)

Browse Source 
* optimize KV cache to reduce GPU memory usage

* revert to using candle_nn::kv_cache::KvCache with initial capacity of 512
This commit is contained in:
Snake
2025-05-13 11:53:42 +08:00
committed by GitHub
parent 36508a2c93
commit 485ddf2996
1 changed files with 3 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
candle-transformers/src/models/quantized_qwen3.rs
View File

@ -160,12 +160,9 @@ impl AttentionWeights {
let q_norm = gg.rms_norm(&format!("{prefix}.attn_q_norm.weight"), rms_norm_eps)?;
let k_norm = gg.rms_norm(&format!("{prefix}.attn_k_norm.weight"), rms_norm_eps)?;
let max_position_embeddings = gg
.metadata()
.get("qwen3.context_length")
.and_then(|v| v.to_u32().ok())
.unwrap_or(4096) as usize;
let kv_cache = KvCache::new(2, max_position_embeddings);
// Initialize KV cache with 512 tokens capacity to reduce initial memory allocation.
// The cache will grow in chunks of 512 tokens when needed.
let kv_cache = KvCache::new(2, 512);
let span_attn = tracing::span!(tracing::Level::TRACE, "attn");