mirror of
https://github.com/huggingface/candle.git
synced 2025-06-19 03:54:56 +00:00
18d30005c5
* Add support to UL2 model family * Update docs with UL2 * Create ActivationWithOptionalGating to avoid polluting activations * Also refactor quantized t5 * Remove useless conversion * Revert Activation::NewGelu name change * Remove useless return * Apply rustfmt and clippy recommendations * Reuse t5::ActivationWithOptionalGating in quantized version * (cosmetic change) use a match rather than ifs + avoid early returns. --------- Co-authored-by: Laurent <laurent.mazare@gmail.com>
43 lines
1.3 KiB
Rust
43 lines
1.3 KiB
Rust
use candle::Tensor;
|
|
use serde::Deserialize;
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Deserialize, Default)]
|
|
#[serde(rename_all = "lowercase")]
|
|
pub enum Activation {
|
|
#[default]
|
|
Gelu,
|
|
NewGelu,
|
|
Relu,
|
|
Relu2,
|
|
Relu6,
|
|
Silu,
|
|
Sigmoid,
|
|
HardSigmoid,
|
|
Swiglu,
|
|
Swish,
|
|
HardSwish,
|
|
Elu(f64),
|
|
LeakyRelu(f64),
|
|
}
|
|
|
|
impl super::Module for Activation {
|
|
fn forward(&self, xs: &Tensor) -> candle::Result<Tensor> {
|
|
match self {
|
|
Self::Gelu => xs.gelu_erf(),
|
|
// https://github.com/huggingface/transformers/blob/12f043eaeaabfef6f6efea411d98e6f6d3c094b7/src/transformers/activations.py#L49-L78
|
|
Self::NewGelu => xs.gelu(),
|
|
Self::Relu => xs.relu(),
|
|
Self::Relu2 => xs.relu()?.sqr(),
|
|
Self::Relu6 => xs.clamp(0f32, 6f32),
|
|
Self::Silu => crate::ops::silu(xs),
|
|
Self::Sigmoid => crate::ops::sigmoid(xs),
|
|
Self::HardSigmoid => crate::ops::hard_sigmoid(xs),
|
|
Self::Swiglu => crate::ops::swiglu(xs),
|
|
Self::Swish => xs * crate::ops::sigmoid(xs)?,
|
|
Self::HardSwish => xs * crate::ops::hard_sigmoid(xs)?,
|
|
&Self::Elu(alpha) => xs.elu(alpha),
|
|
&Self::LeakyRelu(negative_slope) => crate::ops::leaky_relu(xs, negative_slope),
|
|
}
|
|
}
|
|
}
|