huggingface/candle
1
0
Fork 0
mirror of https://github.com/huggingface/candle.git synced 2025-06-15 18:28:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

Updating Add qwen3 (PR 2903) to use HF weights (#2930)

Browse Source 
* add Qwen3.rs

* fixed compile error

* attempting to gett pr 2903 working with qwen weights

* different qwen variants working

* added moe model

* clippy

* added additional eos token

* translated Korean comments to English as well as I can

* removed specialized Qwen3RmsNorm and replaced with generic Candle RmsNorm

* replaced custom repeat_kv implementation with candle's repeat_kv implementation

* replace linear with linear_b in attention initalization

* replaced custom custom kv_cache implementation with candle kv_cache

* style

* replaced explicit broadcast add with normal add in decoder layer

* removed keeping the Rotary embedding layer in the model struct

* used tie_word_embeddings bool from config instead of relying on existence of weights for lm head in CasualLM

* removed duplicate code from qwen3_moe

* removed sliding window from qwen3 attention

* removed MoE code

* removed unused option

* Fixed Typo

Co-authored-by: Laurent Mazare <laurent.mazare@gmail.com>

* fixed tie word embeddings to use the correct embedding weights instead of the opposite

---------

Co-authored-by: Max <naturale@hufs.ac.kr>
Co-authored-by: Laurent Mazare <laurent.mazare@gmail.com>
This commit is contained in:
Kyle Birnbaum
2025-05-01 21:05:53 -07:00
committed by GitHub
parent cd96fa80da
commit 1fdfb58de5
3 changed files with 421 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
candle-examples/examples/qwen/main.rs
View File

@ -9,6 +9,7 @@ use clap::Parser;
use candle_transformers::models::qwen2::{Config as ConfigBase, ModelForCausalLM as ModelBase};
use candle_transformers::models::qwen2_moe::{Config as ConfigMoe, Model as ModelMoe};
use candle_transformers::models::qwen3::{Config as Config3, ModelForCausalLM as Model3};
use candle::{DType, Device, Tensor};
use candle_examples::token_output_stream::TokenOutputStream;
@ -20,6 +21,7 @@ use tokenizers::Tokenizer;
enum Model {
Base(ModelBase),
Moe(ModelMoe),
Base3(Model3),
}
impl Model {
@ -27,6 +29,7 @@ impl Model {
match self {
Self::Moe(ref mut m) => m.forward(xs, s),
Self::Base(ref mut m) => m.forward(xs, s),
Self::Base3(ref mut m) => m.forward(xs, s),
}
}
}
@ -85,6 +88,10 @@ impl TextGeneration {
Some(token) => token,
None => anyhow::bail!("cannot find the <|endoftext|> token"),
};
let eos_token2 = match self.tokenizer.get_token("<|im_end|>") {
Some(token) => token,
None => anyhow::bail!("cannot find the <|im_end|> token"),
};
let start_gen = std::time::Instant::now();
for index in 0..sample_len {
let context_size = if index > 0 { 1 } else { tokens.len() };
@ -107,7 +114,7 @@ impl TextGeneration {
let next_token = self.logits_processor.sample(&logits)?;
tokens.push(next_token);
generated_tokens += 1;
if next_token == eos_token {
if next_token == eos_token || next_token == eos_token2 {
break;
}
if let Some(t) = self.tokenizer.next_token(next_token)? {
@ -152,6 +159,14 @@ enum WhichModel {
W2_7b,
#[value(name = "2-72b")]
W2_72b,
#[value(name = "3-0.6b")]
W3_0_6b,
#[value(name = "3-1.7b")]
W3_1_7b,
#[value(name = "3-4b")]
W3_4b,
#[value(name = "3-8b")]
W3_8b,
}
#[derive(Parser, Debug)]
@ -254,6 +269,10 @@ fn main() -> Result<()> {
WhichModel::W14b => ("1.5", "14B"),
WhichModel::W72b => ("1.5", "72B"),
WhichModel::MoeA27b => ("1.5", "MoE-A2.7B"),
WhichModel::W3_0_6b => ("3", "0.6B"),
WhichModel::W3_1_7b => ("3", "1.7B"),
WhichModel::W3_4b => ("3", "4B"),
WhichModel::W3_8b => ("3", "8B"),
};
format!("Qwen/Qwen{version}-{size}")
}
@ -273,7 +292,11 @@ fn main() -> Result<()> {
.map(std::path::PathBuf::from)
.collect::<Vec<_>>(),
None => match args.model {
WhichModel::W0_5b | WhichModel::W2_0_5b | WhichModel::W2_1_5b | WhichModel::W1_8b => {
WhichModel::W0_5b
| WhichModel::W2_0_5b
| WhichModel::W2_1_5b
| WhichModel::W1_8b
| WhichModel::W3_0_6b => {
vec![repo.get("model.safetensors")?]
}
WhichModel::W4b
@ -282,7 +305,10 @@ fn main() -> Result<()> {
| WhichModel::W14b
| WhichModel::W72b
| WhichModel::W2_72b
| WhichModel::MoeA27b => {
| WhichModel::MoeA27b
| WhichModel::W3_1_7b
| WhichModel::W3_4b
| WhichModel::W3_8b => {
candle_examples::hub_load_safetensors(&repo, "model.safetensors.index.json")?
}
},
@ -304,6 +330,10 @@ fn main() -> Result<()> {
let config: ConfigMoe = serde_json::from_slice(&std::fs::read(config_file)?)?;
Model::Moe(ModelMoe::new(&config, vb)?)
}
WhichModel::W3_0_6b | WhichModel::W3_1_7b | WhichModel::W3_4b | WhichModel::W3_8b => {
let config: Config3 = serde_json::from_slice(&std::fs::read(config_file)?)?;
Model::Base3(Model3::new(&config, vb)?)
}
_ => {
let config: ConfigBase = serde_json::from_slice(&std::fs::read(config_file)?)?;
Model::Base(ModelBase::new(&config, vb)?)