From 59bbc0d28775d3bc3224520bdeb4459a90a1eaba Mon Sep 17 00:00:00 2001
From: Laurent Mazare <laurent.mazare@gmail.com>
Date: Mon, 5 Aug 2024 20:03:31 +0100
Subject: [PATCH] Add the import script for the T5 tokenizer. (#2399)

---
 candle-examples/examples/flux/t5_tokenizer.py | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100644 candle-examples/examples/flux/t5_tokenizer.py

diff --git a/candle-examples/examples/flux/t5_tokenizer.py b/candle-examples/examples/flux/t5_tokenizer.py
new file mode 100644
index 00000000..cebd0624
--- /dev/null
+++ b/candle-examples/examples/flux/t5_tokenizer.py
@@ -0,0 +1,6 @@
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+BASE_MODEL = "google/t5-v1_1-xxl"
+tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL)
+# The tokenizer will be saved in /tmp/tokenizer/tokenizer.json
+tokenizer.save_pretrained("/tmp/tokenizer/")