From 8ede5f42102326a6c0e1cf7b824d781018ad6350 Mon Sep 17 00:00:00 2001 From: Daniel Clough <9276072+danielclough@users.noreply.github.com> Date: Wed, 20 Dec 2023 12:03:24 -0800 Subject: [PATCH] add fn config_chat_ml (#1458) * add fn config_chat_ml * Add a link to the original config. --------- Co-authored-by: Ubuntu Co-authored-by: laurent --- candle-transformers/src/models/mistral.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/candle-transformers/src/models/mistral.rs b/candle-transformers/src/models/mistral.rs index caf96bce..0ba556bd 100644 --- a/candle-transformers/src/models/mistral.rs +++ b/candle-transformers/src/models/mistral.rs @@ -21,6 +21,7 @@ pub struct Config { } impl Config { + // https://huggingface.co/mistralai/Mistral-7B-v0.1/blob/main/config.json pub fn config_7b_v0_1(use_flash_attn: bool) -> Self { Self { vocab_size: 32000, @@ -37,6 +38,24 @@ impl Config { use_flash_attn, } } + + // https://huggingface.co/Open-Orca/Mistral-7B-OpenOrca/blob/main/config.json + pub fn config_open_orca_chat_ml(use_flash_attn: bool) -> Self { + Self { + vocab_size: 32002, + hidden_size: 4096, + intermediate_size: 14336, + num_hidden_layers: 32, + num_attention_heads: 32, + num_key_value_heads: 8, + hidden_act: Activation::Silu, + max_position_embeddings: 32768, + rms_norm_eps: 1e-5, + rope_theta: 10_000., + sliding_window: 4096, + use_flash_attn, + } + } } #[derive(Debug, Clone)]