huggingface/candle
1
0
Fork 0
mirror of https://github.com/huggingface/candle.git synced 2025-06-16 18:48:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add hard-sigmoid and hard-swish activations (#1244)

Browse Source 
* Add hard-sigmoid and hard-swish activations

* Update ops.rs

* Use / rather than div.

---------

Co-authored-by: Laurent <laurent.mazare@gmail.com>
This commit is contained in:
jamjamjon
2023-11-03 01:20:27 +08:00
committed by GitHub
parent b97463098c
commit d39d0c40fd
2 changed files with 9 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
candle-nn/src/activation.rs
View File

@ -13,7 +13,9 @@ pub enum Activation {
Relu6, Relu6,
Silu, Silu,
Sigmoid, Sigmoid,
HardSigmoid,
Swish, Swish,
HardSwish,
Elu(f64), Elu(f64),
LeakyRelu(f64), LeakyRelu(f64),
} }
@ -29,7 +31,9 @@ impl super::Module for Activation {
Self::Relu6 => xs.clamp(0f32, 6f32), Self::Relu6 => xs.clamp(0f32, 6f32),
Self::Silu => crate::ops::silu(xs), Self::Silu => crate::ops::silu(xs),
Self::Sigmoid => crate::ops::sigmoid(xs), Self::Sigmoid => crate::ops::sigmoid(xs),
Self::HardSigmoid => crate::ops::hard_sigmoid(xs),
Self::Swish => xs * crate::ops::sigmoid(xs)?, Self::Swish => xs * crate::ops::sigmoid(xs)?,
Self::HardSwish => xs * crate::ops::hard_sigmoid(xs)?,
&Self::Elu(alpha) => xs.elu(alpha), &Self::Elu(alpha) => xs.elu(alpha),
&Self::LeakyRelu(negative_slope) => crate::ops::leaky_relu(xs, negative_slope), &Self::LeakyRelu(negative_slope) => crate::ops::leaky_relu(xs, negative_slope),
} }

5
candle-nn/src/ops.rs
View File

@ -44,6 +44,11 @@ pub fn sigmoid(xs: &Tensor) -> Result<Tensor> {
(xs.neg()?.exp()? + 1.0)?.recip() (xs.neg()?.exp()? + 1.0)?.recip()
} }
pub fn hard_sigmoid(xs: &Tensor) -> Result<Tensor> {
// TODO: Should we have a specialized op for this?
((xs + 3.0)? / 6.0)?.clamp(0f32, 1f32)
}
pub fn leaky_relu(xs: &Tensor, negative_slope: f64) -> Result<Tensor> { pub fn leaky_relu(xs: &Tensor, negative_slope: f64) -> Result<Tensor> {
let zeros = xs.zeros_like()?; let zeros = xs.zeros_like()?;
xs.maximum(&zeros)? + xs.minimum(&zeros)? * negative_slope xs.maximum(&zeros)? + xs.minimum(&zeros)? * negative_slope