huggingface/candle
1
0
Fork 0
mirror of https://github.com/huggingface/candle.git synced 2025-06-16 18:48:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

Use a silu activation in mistral. (#991)

Browse Source
This commit is contained in:
Laurent Mazare
2023-09-29 08:06:54 +02:00
committed by GitHub
parent 23b3576c47
commit 53510ce427
2 changed files with 5 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
candle-nn/src/activation.rs
View File

@ -9,6 +9,8 @@ pub enum Activation {
#[serde(rename = "gated-gelu")] #[serde(rename = "gated-gelu")]
NewGelu, NewGelu,
Relu, Relu,
Silu,
Sigmoid,
Elu(f64), Elu(f64),
LeakyRelu(f64), LeakyRelu(f64),
} }
@ -20,6 +22,8 @@ impl super::Module for Activation {
// https://github.com/huggingface/transformers/blob/12f043eaeaabfef6f6efea411d98e6f6d3c094b7/src/transformers/activations.py#L49-L78 // https://github.com/huggingface/transformers/blob/12f043eaeaabfef6f6efea411d98e6f6d3c094b7/src/transformers/activations.py#L49-L78
Self::NewGelu => xs.gelu(), Self::NewGelu => xs.gelu(),
Self::Relu => xs.relu(), Self::Relu => xs.relu(),
Self::Silu => crate::ops::silu(xs),
Self::Sigmoid => crate::ops::sigmoid(xs),
&Self::Elu(alpha) => xs.elu(alpha), &Self::Elu(alpha) => xs.elu(alpha),
&Self::LeakyRelu(negative_slope) => crate::ops::leaky_relu(xs, negative_slope), &Self::LeakyRelu(negative_slope) => crate::ops::leaky_relu(xs, negative_slope),
} }

2
candle-transformers/src/models/mistral.rs
View File

@ -28,7 +28,7 @@ impl Config {
num_hidden_layers: 32, num_hidden_layers: 32,
num_attention_heads: 32, num_attention_heads: 32,
num_key_value_heads: 8, num_key_value_heads: 8,
hidden_act: Activation::Gelu, // TODO: silu hidden_act: Activation::Silu,
max_position_embeddings: 32768, max_position_embeddings: 32768,
rms_norm_eps: 1e-5, rms_norm_eps: 1e-5,
rope_theta: 10_000., rope_theta: 10_000.,