huggingface/candle
1
0
Fork 0
mirror of https://github.com/huggingface/candle.git synced 2025-06-20 12:06:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Use BF16 for llama v3 by default.

Browse Source
This commit is contained in:
laurent
2024-04-19 14:22:01 +02:00
parent 9c532aef47
commit 6d6d87f8b3
1 changed files with 4 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
candle-examples/examples/llama/main.rs
View File

@ -120,7 +120,10 @@ fn main() -> Result<()> {
Some("bf16") => DType::BF16, Some("bf16") => DType::BF16,
Some("f32") => DType::F32, Some("f32") => DType::F32,
Some(dtype) => bail!("Unsupported dtype {dtype}"), Some(dtype) => bail!("Unsupported dtype {dtype}"),
None => DType::F16, None => match args.which {
Which::V3 | Which::V3Instruct => DType::BF16,
Which::V1 | Which::V2 | Which::Solar10_7B | Which::TinyLlama1_1BChat => DType::F16,
},
}; };
let (llama, tokenizer_filename, mut cache, config) = { let (llama, tokenizer_filename, mut cache, config) = {
let api = Api::new()?; let api = Api::new()?;