huggingface/candle
1
0
Fork 0
mirror of https://github.com/huggingface/candle.git synced 2025-06-16 10:38:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Switch the default to using the faster kernels. (#1978)

Browse Source 
* Switch the default to using the faster kernels.

* Add the force-dmmv flag.
This commit is contained in:
Laurent Mazare
2024-04-01 10:00:11 +02:00
committed by GitHub
parent cd29c7ccd4
commit c7557b65dc
3 changed files with 11 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
candle-examples/examples/quantized/main.rs
View File

@ -236,9 +236,9 @@ struct Args {
#[arg(long)]
gqa: Option<usize>,
/// Use the (experimental) fast cuda kernels.
/// Use the slower dmmv cuda kernel.
#[arg(long)]
fast_cuda: bool,
force_dmmv: bool,
}
impl Args {
@ -347,7 +347,7 @@ fn main() -> anyhow::Result<()> {
let args = Args::parse();
#[cfg(feature = "cuda")]
candle::quantized::cuda::set_force_dmmv(!args.fast_cuda);
candle::quantized::cuda::set_force_dmmv(args.force_dmmv);
let temperature = if args.temperature == 0. {
None