mirror of
https://github.com/huggingface/candle.git
synced 2025-06-16 18:48:51 +00:00
7 lines
258 B
Python
7 lines
258 B
Python
from transformers import AutoModelForCausalLM, AutoTokenizer
|
|
|
|
BASE_MODEL = "google/t5-v1_1-xxl"
|
|
tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL)
|
|
# The tokenizer will be saved in /tmp/tokenizer/tokenizer.json
|
|
tokenizer.save_pretrained("/tmp/tokenizer/")
|