mirror of
https://github.com/huggingface/candle.git
synced 2025-06-16 10:38:54 +00:00
7c2449f623
* Improve reduce perf and add contiguous impl * Improve arg reduce and add contiguous impl * Improve softmax kernel. 33%-39% higher thrpt * fmt * Fixed all bugs. Improved code quality. Added tests. * Stash for debugging * Stash for debugging 2 * Fixing argmax bug and improve performance Co-authored-by: Christopher Fleetwood <45471420+FL33TW00D@users.noreply.github.com> * Fix test and add is_valid_simgroup_reduce_type trait * Online softmax. Improved threadgroup reduce. Tidying up a bit. * Remove redundant threadgroup_barrier from arg reduce * Mostly tidying up. Some improvements * Simplify indexed struct * tidying * Reuse operation operator instead of passing it in as a parameter * Fix how operators are applied to indexed<vec<T,N>> * Vectorized load. Scalar block reduce. Hitting max throughput for f32 reduce. * Vectorized load for online softmax. Involves a reinterpret_cast of src which may be suboptimal. * Metal as_type casting vec<bfloat, N> -> vec<float, N/2> for simd and fast math * Use constant for input instead of const device. Fix strided reduce. * Use contiguous reduce in tests * Rename finalize -> to_scalar * Support integer types max/min (switch with trait-inferred impl later) * Was worried I was skipping work -> shuffling the 1D test cases * Add build.rs to avoid metal kernel jit compile overhead * Improve build. Extract utils * Compile metal kernels for both macos and ios * Fixed over xmas and then forgot about it * Add calculate_reduce_threads util * Remove old reduce.metal * Improve f16/bf16 softmax precision by accumulating in f32 * Remove build.rs (for now) * Move softmax bench to candle-nn * Remove redundant thread calc util fn * Use uint over ushort for indices etc * Use fast exp in MDReduceOp * Remove nested metal define for softmax * Fix some clippy lint. --------- Co-authored-by: Christopher Fleetwood <45471420+FL33TW00D@users.noreply.github.com> Co-authored-by: Laurent <laurent.mazare@gmail.com>
71 lines
2.0 KiB
Rust
71 lines
2.0 KiB
Rust
pub(crate) mod affine;
|
|
pub(crate) mod conv_transpose2d;
|
|
pub(crate) mod matmul;
|
|
pub(crate) mod qmatmul;
|
|
pub(crate) mod random;
|
|
pub(crate) mod reduce;
|
|
pub(crate) mod unary;
|
|
pub(crate) mod where_cond;
|
|
|
|
use candle_core::{Device, Result};
|
|
|
|
pub(crate) trait BenchDevice {
|
|
fn sync(&self) -> Result<()>;
|
|
|
|
fn bench_name<S: Into<String>>(&self, name: S) -> String;
|
|
}
|
|
|
|
impl BenchDevice for Device {
|
|
fn sync(&self) -> Result<()> {
|
|
match self {
|
|
Device::Cpu => Ok(()),
|
|
Device::Cuda(device) => {
|
|
#[cfg(feature = "cuda")]
|
|
return Ok(device.synchronize()?);
|
|
#[cfg(not(feature = "cuda"))]
|
|
panic!("Cuda device without cuda feature enabled: {:?}", device)
|
|
}
|
|
Device::Metal(device) => {
|
|
#[cfg(feature = "metal")]
|
|
return Ok(device.wait_until_completed()?);
|
|
#[cfg(not(feature = "metal"))]
|
|
panic!("Metal device without metal feature enabled: {:?}", device)
|
|
}
|
|
}
|
|
}
|
|
|
|
fn bench_name<S: Into<String>>(&self, name: S) -> String {
|
|
match self {
|
|
Device::Cpu => {
|
|
let cpu_type = if cfg!(feature = "accelerate") {
|
|
"accelerate"
|
|
} else if cfg!(feature = "mkl") {
|
|
"mkl"
|
|
} else {
|
|
"cpu"
|
|
};
|
|
format!("{}_{}", cpu_type, name.into())
|
|
}
|
|
Device::Cuda(_) => format!("cuda_{}", name.into()),
|
|
Device::Metal(_) => format!("metal_{}", name.into()),
|
|
}
|
|
}
|
|
}
|
|
|
|
struct BenchDeviceHandler {
|
|
devices: Vec<Device>,
|
|
}
|
|
|
|
impl BenchDeviceHandler {
|
|
pub fn new() -> Result<Self> {
|
|
let mut devices = Vec::new();
|
|
if cfg!(feature = "metal") {
|
|
devices.push(Device::new_metal(0)?);
|
|
} else if cfg!(feature = "cuda") {
|
|
devices.push(Device::new_cuda(0)?);
|
|
}
|
|
devices.push(Device::Cpu);
|
|
Ok(Self { devices })
|
|
}
|
|
}
|