huggingface/candle
1
0
Fork 0
mirror of https://github.com/huggingface/candle.git synced 2025-06-16 10:38:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Use the gelu-erf activation. (#969)

Browse Source
This commit is contained in:
Laurent Mazare
2023-09-26 22:30:21 +01:00
committed by GitHub
parent 4abc1ea34d
commit ce0a4e3a85
3 changed files with 5 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
candle-nn/src/activation.rs
View File

@ -16,9 +16,7 @@ pub enum Activation {
impl super::Module for Activation {
fn forward(&self, xs: &Tensor) -> candle::Result<Tensor> {
match self {
Self::Gelu => xs.gelu(),
// TODO: This is "gelu_new", not the original "gelu".
// There's some small numerical difference:
Self::Gelu => xs.gelu_erf(),
// https://github.com/huggingface/transformers/blob/12f043eaeaabfef6f6efea411d98e6f6d3c094b7/src/transformers/activations.py#L49-L78
Self::NewGelu => xs.gelu(),
Self::Relu => xs.relu(),