Use a silu activation in mistral. (#991)

2025-06-16 02:38:10 +00:00 · 2023-09-29 08:06:54 +02:00
parent 23b3576c47
commit 53510ce427
2 changed files with 5 additions and 1 deletions
--- a/candle-transformers/src/models/mistral.rs
+++ b/candle-transformers/src/models/mistral.rs
@ -28,7 +28,7 @@ impl Config {
            num_hidden_layers: 32,
            num_attention_heads: 32,
            num_key_value_heads: 8,
-            hidden_act: Activation::Gelu, // TODO: silu
+            hidden_act: Activation::Silu,
            max_position_embeddings: 32768,
            rms_norm_eps: 1e-5,
            rope_theta: 10_000.,