huggingface/candle
1
0
Fork 0
mirror of https://github.com/huggingface/candle.git synced 2025-06-18 19:47:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add support to flan-t5 (#840)

Browse Source
This commit is contained in:
Juarez Bochi
2023-09-13 10:27:20 -07:00
committed by GitHub
parent 9a465e1b26
commit 49d3f7f708
2 changed files with 55 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
candle-nn/src/activation.rs
View File

@ -6,6 +6,8 @@ use serde::Deserialize;
pub enum Activation {
#[default]
Gelu,
#[serde(rename = "gated-gelu")]
NewGelu,
Relu,
Elu(f64),
}
@ -14,6 +16,10 @@ impl super::Module for Activation {
fn forward(&self, xs: &Tensor) -> candle::Result<Tensor> {
match self {
Self::Gelu => xs.gelu(),
// TODO: This is "gelu_new", not the original "gelu".
// There's some small numerical difference:
// https://github.com/huggingface/transformers/blob/12f043eaeaabfef6f6efea411d98e6f6d3c094b7/src/transformers/activations.py#L49-L78
Self::NewGelu => xs.gelu(),
Self::Relu => xs.relu(),
&Self::Elu(alpha) => xs.elu(alpha),
}