From cd889c0f8a0a4aecd5ed6268a12394cd2462c202 Mon Sep 17 00:00:00 2001 From: Daniel Clough <9276072+danielclough@users.noreply.github.com> Date: Thu, 28 Dec 2023 10:59:58 -0800 Subject: [PATCH] add config_amazon_mistral_lite (#1493) Co-authored-by: Ubuntu --- candle-transformers/src/models/mistral.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/candle-transformers/src/models/mistral.rs b/candle-transformers/src/models/mistral.rs index 2a66515b..2809ae0a 100644 --- a/candle-transformers/src/models/mistral.rs +++ b/candle-transformers/src/models/mistral.rs @@ -57,6 +57,24 @@ impl Config { use_flash_attn, } } + + // https://huggingface.co/amazon/MistralLite/blob/main/config.json + pub fn config_amazon_mistral_lite(use_flash_attn: bool) -> Self { + Self { + vocab_size: 32003, + hidden_size: 4096, + intermediate_size: 14336, + num_hidden_layers: 32, + num_attention_heads: 32, + num_key_value_heads: 8, + hidden_act: Activation::Silu, + max_position_embeddings: 32768, + rms_norm_eps: 1e-5, + rope_theta: 10_000., + sliding_window: 4096, + use_flash_attn, + } + } } #[derive(Debug, Clone)]