huggingface/candle
1
0
Fork 0
mirror of https://github.com/huggingface/candle.git synced 2025-06-17 19:18:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Use the gelu-erf activation. (#969)

Browse Source
This commit is contained in:
Laurent Mazare
2023-09-26 22:30:21 +01:00
committed by GitHub
parent 4abc1ea34d
commit ce0a4e3a85
3 changed files with 5 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
candle-transformers/src/models/bert.rs
View File

@ -25,10 +25,8 @@ impl HiddenActLayer {
fn forward(&self, xs: &Tensor) -> candle::Result<Tensor> {
let _enter = self.span.enter();
match self.act {
// TODO: The all-MiniLM-L6-v2 model uses "gelu" whereas this is "gelu_new", this explains some
// small numerical difference.
// https://github.com/huggingface/transformers/blob/cd4584e3c809bb9e1392ccd3fe38b40daba5519a/src/transformers/activations.py#L213
HiddenAct::Gelu => xs.gelu(),
HiddenAct::Gelu => xs.gelu_erf(),
HiddenAct::Relu => xs.relu(),
}
}