mirror of
https://github.com/huggingface/candle.git
synced 2025-06-15 10:26:33 +00:00
577 lines
19 KiB
HTML
577 lines
19 KiB
HTML
<html>
|
||
<head>
|
||
<meta content="text/html;charset=utf-8" http-equiv="Content-Type" />
|
||
<title>Candle Phi 1.5 / Phi 2.0 Rust/WASM</title>
|
||
</head>
|
||
<body></body>
|
||
</html>
|
||
|
||
<!DOCTYPE html>
|
||
<html>
|
||
<head>
|
||
<meta charset="UTF-8" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||
<link
|
||
rel="stylesheet"
|
||
href="https://cdn.jsdelivr.net/gh/highlightjs/cdn-release@11.8.0/build/styles/default.min.css"
|
||
/>
|
||
<style>
|
||
@import url("https://fonts.googleapis.com/css2?family=Source+Code+Pro:wght@200;300;400&family=Source+Sans+3:wght@100;200;300;400;500;600;700;800;900&display=swap");
|
||
html,
|
||
body {
|
||
font-family: "Source Sans 3", sans-serif;
|
||
}
|
||
code,
|
||
output,
|
||
select,
|
||
pre {
|
||
font-family: "Source Code Pro", monospace;
|
||
}
|
||
</style>
|
||
<style type="text/tailwindcss">
|
||
.link {
|
||
@apply underline hover:text-blue-500 hover:no-underline;
|
||
}
|
||
</style>
|
||
<script src="https://cdn.tailwindcss.com"></script>
|
||
<script type="module">
|
||
import snarkdown from "https://cdn.skypack.dev/snarkdown";
|
||
import hljs from "https://cdn.skypack.dev/highlight.js";
|
||
// models base url
|
||
const MODELS = {
|
||
phi_1_5_q4k: {
|
||
base_url:
|
||
"https://huggingface.co/lmz/candle-quantized-phi/resolve/main/",
|
||
model: "model-q4k.gguf",
|
||
tokenizer: "tokenizer.json",
|
||
config: "phi-1_5.json",
|
||
quantized: true,
|
||
seq_len: 2048,
|
||
size: "800 MB",
|
||
},
|
||
phi_1_5_q80: {
|
||
base_url:
|
||
"https://huggingface.co/lmz/candle-quantized-phi/resolve/main/",
|
||
model: "model-q80.gguf",
|
||
tokenizer: "tokenizer.json",
|
||
config: "phi-1_5.json",
|
||
quantized: true,
|
||
seq_len: 2048,
|
||
size: "1.51 GB",
|
||
},
|
||
phi_2_0_q4k: {
|
||
base_url:
|
||
"https://huggingface.co/radames/phi-2-quantized/resolve/main/",
|
||
model: [
|
||
"model-v2-q4k.gguf_aa.part",
|
||
"model-v2-q4k.gguf_ab.part",
|
||
"model-v2-q4k.gguf_ac.part",
|
||
],
|
||
tokenizer: "tokenizer.json",
|
||
config: "config.json",
|
||
quantized: true,
|
||
seq_len: 2048,
|
||
size: "1.57GB",
|
||
},
|
||
puffin_phi_v2_q4k: {
|
||
base_url:
|
||
"https://huggingface.co/lmz/candle-quantized-phi/resolve/main/",
|
||
model: "model-puffin-phi-v2-q4k.gguf",
|
||
tokenizer: "tokenizer-puffin-phi-v2.json",
|
||
config: "puffin-phi-v2.json",
|
||
quantized: true,
|
||
seq_len: 2048,
|
||
size: "798 MB",
|
||
},
|
||
puffin_phi_v2_q80: {
|
||
base_url:
|
||
"https://huggingface.co/lmz/candle-quantized-phi/resolve/main/",
|
||
model: "model-puffin-phi-v2-q80.gguf",
|
||
tokenizer: "tokenizer-puffin-phi-v2.json",
|
||
config: "puffin-phi-v2.json",
|
||
quantized: true,
|
||
seq_len: 2048,
|
||
size: "1.50 GB",
|
||
},
|
||
};
|
||
|
||
const TEMPLATES = [
|
||
{
|
||
title: "Simple prompt",
|
||
prompt: `Sebastien is in London today, it’s the middle of July yet it’s raining, so Sebastien is feeling gloomy. He`,
|
||
},
|
||
{
|
||
title: "Think step by step",
|
||
prompt: `Suppose Alice originally had 3 apples, then Bob gave Alice 7 apples, then Alice gave Cook 5 apples, and then Tim gave Alice 3x the amount of apples Alice had. How many apples does Alice have now?
|
||
Let’s think step by step.`,
|
||
},
|
||
{
|
||
title: "Explaing a code snippet",
|
||
prompt: `What does this script do?
|
||
\`\`\`python
|
||
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||
s.bind(('', 0))
|
||
s.listen(1)
|
||
conn, addr = s.accept()
|
||
print('Connected by', addr)
|
||
return conn.getsockname()[1]
|
||
\`\`\`
|
||
Let’s think step by step.`,
|
||
},
|
||
{
|
||
title: "Question answering",
|
||
prompt: `Instruct: What is the capital of France?
|
||
Output:`,
|
||
},
|
||
{
|
||
title: "Chat mode",
|
||
prompt: `Alice: Can you tell me how to create a python application to go through all the files
|
||
in one directory where the file’s name DOES NOT end with '.json'?
|
||
Bob:`,
|
||
},
|
||
{
|
||
title: "Python code completion",
|
||
prompt: `"""write a python function called batch(function, list) which call function(x) for x in
|
||
list in parallel"""
|
||
Solution:`,
|
||
},
|
||
{
|
||
title: "Python Sample",
|
||
prompt: `"""Can you make sure those histograms appear side by side on the same plot:
|
||
\`\`\`python
|
||
plt.hist(intreps_retrained[0][1].view(64,-1).norm(dim=1).detach().cpu().numpy(), bins = 20)
|
||
plt.hist(intreps_pretrained[0][1].view(64,-1).norm(dim=1).detach().cpu().numpy(), bins = 20)
|
||
\`\`\`
|
||
"""`,
|
||
},
|
||
{
|
||
title: "Write a Twitter post",
|
||
prompt: `Write a twitter post for the discovery of gravitational wave.
|
||
Twitter Post:`,
|
||
},
|
||
{
|
||
title: "Write a review",
|
||
prompt: `Write a polite review complaining that the video game 'Random Game' was too badly optimized and it burned my laptop.
|
||
Very polite review:`,
|
||
},
|
||
];
|
||
const phiWorker = new Worker("./phiWorker.js", {
|
||
type: "module",
|
||
});
|
||
async function generateSequence(controller) {
|
||
const getValue = (id) => document.querySelector(`#${id}`).value;
|
||
const modelID = getValue("model");
|
||
const model = MODELS[modelID];
|
||
const weightsURL =
|
||
model.model instanceof Array
|
||
? model.model.map((m) => model.base_url + m)
|
||
: model.base_url + model.model;
|
||
const tokenizerURL = model.base_url + model.tokenizer;
|
||
const configURL = model.base_url + model.config;
|
||
|
||
const prompt = getValue("prompt").trim();
|
||
const temperature = getValue("temperature");
|
||
const topP = getValue("top-p");
|
||
const repeatPenalty = getValue("repeat_penalty");
|
||
const seed = getValue("seed");
|
||
const maxSeqLen = getValue("max-seq");
|
||
|
||
function updateStatus(data) {
|
||
const outStatus = document.querySelector("#output-status");
|
||
const outGen = document.querySelector("#output-generation");
|
||
const outCounter = document.querySelector("#output-counter");
|
||
|
||
switch (data.status) {
|
||
case "loading":
|
||
outStatus.hidden = false;
|
||
outStatus.textContent = data.message;
|
||
outGen.hidden = true;
|
||
outCounter.hidden = true;
|
||
break;
|
||
case "generating":
|
||
const { message, prompt, sentence, tokensSec, totalTime } = data;
|
||
outStatus.hidden = true;
|
||
outCounter.hidden = false;
|
||
outGen.hidden = false;
|
||
outGen.innerHTML = snarkdown(prompt + sentence);
|
||
outCounter.innerHTML = `${(totalTime / 1000).toFixed(
|
||
2
|
||
)}s (${tokensSec.toFixed(2)} tok/s)`;
|
||
hljs.highlightAll();
|
||
break;
|
||
case "complete":
|
||
outStatus.hidden = true;
|
||
outGen.hidden = false;
|
||
break;
|
||
}
|
||
}
|
||
|
||
return new Promise((resolve, reject) => {
|
||
phiWorker.postMessage({
|
||
weightsURL,
|
||
modelID,
|
||
tokenizerURL,
|
||
configURL,
|
||
quantized: model.quantized,
|
||
prompt,
|
||
temp: temperature,
|
||
top_p: topP,
|
||
repeatPenalty,
|
||
seed: seed,
|
||
maxSeqLen,
|
||
command: "start",
|
||
});
|
||
|
||
const handleAbort = () => {
|
||
phiWorker.postMessage({ command: "abort" });
|
||
};
|
||
const handleMessage = (event) => {
|
||
const { status, error, message, prompt, sentence } = event.data;
|
||
if (status) updateStatus(event.data);
|
||
if (error) {
|
||
phiWorker.removeEventListener("message", handleMessage);
|
||
reject(new Error(error));
|
||
}
|
||
if (status === "aborted") {
|
||
phiWorker.removeEventListener("message", handleMessage);
|
||
resolve(event.data);
|
||
}
|
||
if (status === "complete") {
|
||
phiWorker.removeEventListener("message", handleMessage);
|
||
resolve(event.data);
|
||
}
|
||
};
|
||
|
||
controller.signal.addEventListener("abort", handleAbort);
|
||
phiWorker.addEventListener("message", handleMessage);
|
||
});
|
||
}
|
||
|
||
const form = document.querySelector("#form");
|
||
const prompt = document.querySelector("#prompt");
|
||
const clearBtn = document.querySelector("#clear-btn");
|
||
const runBtn = document.querySelector("#run");
|
||
const modelSelect = document.querySelector("#model");
|
||
const promptTemplates = document.querySelector("#prompt-templates");
|
||
let runController = new AbortController();
|
||
let isRunning = false;
|
||
|
||
document.addEventListener("DOMContentLoaded", () => {
|
||
for (const [id, model] of Object.entries(MODELS)) {
|
||
const option = document.createElement("option");
|
||
option.value = id;
|
||
option.innerText = `${id} (${model.size})`;
|
||
modelSelect.appendChild(option);
|
||
}
|
||
const query = new URLSearchParams(window.location.search);
|
||
const modelID = query.get("model");
|
||
if (modelID) {
|
||
modelSelect.value = modelID;
|
||
} else {
|
||
modelSelect.value = "phi_1_5_q4k";
|
||
}
|
||
|
||
for (const [i, { title, prompt }] of TEMPLATES.entries()) {
|
||
const div = document.createElement("div");
|
||
const input = document.createElement("input");
|
||
input.type = "radio";
|
||
input.name = "task";
|
||
input.id = `templates-${i}`;
|
||
input.classList.add("font-light", "cursor-pointer");
|
||
input.value = prompt;
|
||
const label = document.createElement("label");
|
||
label.htmlFor = `templates-${i}`;
|
||
label.classList.add("cursor-pointer");
|
||
label.innerText = title;
|
||
div.appendChild(input);
|
||
div.appendChild(label);
|
||
promptTemplates.appendChild(div);
|
||
}
|
||
});
|
||
|
||
promptTemplates.addEventListener("change", (e) => {
|
||
const template = e.target.value;
|
||
prompt.value = template;
|
||
prompt.style.height = "auto";
|
||
prompt.style.height = prompt.scrollHeight + "px";
|
||
runBtn.disabled = false;
|
||
clearBtn.classList.remove("invisible");
|
||
});
|
||
modelSelect.addEventListener("change", (e) => {
|
||
const query = new URLSearchParams(window.location.search);
|
||
query.set("model", e.target.value);
|
||
window.history.replaceState(
|
||
{},
|
||
"",
|
||
`${window.location.pathname}?${query}`
|
||
);
|
||
window.parent.postMessage({ queryString: "?" + query }, "*");
|
||
const model = MODELS[e.target.value];
|
||
document.querySelector("#max-seq").max = model.seq_len;
|
||
document.querySelector("#max-seq").nextElementSibling.value = 200;
|
||
});
|
||
|
||
form.addEventListener("submit", async (e) => {
|
||
e.preventDefault();
|
||
if (isRunning) {
|
||
stopRunning();
|
||
} else {
|
||
startRunning();
|
||
await generateSequence(runController);
|
||
stopRunning();
|
||
}
|
||
});
|
||
|
||
function startRunning() {
|
||
isRunning = true;
|
||
runBtn.textContent = "Stop";
|
||
}
|
||
|
||
function stopRunning() {
|
||
runController.abort();
|
||
runController = new AbortController();
|
||
runBtn.textContent = "Run";
|
||
isRunning = false;
|
||
}
|
||
clearBtn.addEventListener("click", (e) => {
|
||
e.preventDefault();
|
||
prompt.value = "";
|
||
clearBtn.classList.add("invisible");
|
||
runBtn.disabled = true;
|
||
stopRunning();
|
||
});
|
||
prompt.addEventListener("input", (e) => {
|
||
runBtn.disabled = false;
|
||
if (e.target.value.length > 0) {
|
||
clearBtn.classList.remove("invisible");
|
||
} else {
|
||
clearBtn.classList.add("invisible");
|
||
}
|
||
});
|
||
</script>
|
||
</head>
|
||
<body class="container max-w-4xl mx-auto p-4 text-gray-800">
|
||
<main class="grid grid-cols-1 gap-8 relative">
|
||
<span class="absolute text-5xl -ml-[1em]"> 🕯️ </span>
|
||
<div>
|
||
<h1 class="text-5xl font-bold">Candle Phi 1.5 / Phi 2.0</h1>
|
||
<h2 class="text-2xl font-bold">Rust/WASM Demo</h2>
|
||
<p class="max-w-lg">
|
||
The
|
||
<a
|
||
href="https://huggingface.co/microsoft/phi-1_5"
|
||
class="link"
|
||
target="_blank"
|
||
>Phi-1.5</a
|
||
>
|
||
and
|
||
<a
|
||
href="https://huggingface.co/microsoft/phi-2"
|
||
class="link"
|
||
target="_blank"
|
||
>Phi-2</a
|
||
>
|
||
models achieve state-of-the-art performance with only 1.3 billion and
|
||
2.7 billion parameters, compared to larger models with up to 13
|
||
billion parameters. Here you can try the quantized versions.
|
||
Additional prompt examples are available in the
|
||
<a
|
||
href="https://arxiv.org/pdf/2309.05463.pdf#page=8"
|
||
class="link"
|
||
target="_blank"
|
||
>
|
||
technical report </a
|
||
>.
|
||
</p>
|
||
<p class="max-w-lg">
|
||
You can also try
|
||
<a
|
||
href="https://huggingface.co/teknium/Puffin-Phi-v2"
|
||
class="link"
|
||
target="_blank"
|
||
>Puffin-Phi V2
|
||
</a>
|
||
quantized version, a fine-tuned version of Phi-1.5 on the
|
||
<a
|
||
href="https://huggingface.co/datasets/LDJnr/Puffin"
|
||
class="link"
|
||
target="_blank"
|
||
>Puffin dataset
|
||
</a>
|
||
</p>
|
||
</div>
|
||
<div>
|
||
<p class="text-xs italic max-w-lg">
|
||
<b>Note:</b>
|
||
When first run, the app will download and cache the model, which could
|
||
take a few minutes. The models are <b>~800MB</b> or <b>~1.57GB</b> in
|
||
size.
|
||
</p>
|
||
</div>
|
||
<div>
|
||
<label for="model" class="font-medium">Models Options: </label>
|
||
<select
|
||
id="model"
|
||
class="border-2 border-gray-500 rounded-md font-light"
|
||
></select>
|
||
</div>
|
||
<div>
|
||
<details>
|
||
<summary class="font-medium cursor-pointer">Prompt Templates</summary>
|
||
<form
|
||
id="prompt-templates"
|
||
class="grid grid-cols-1 sm:grid-cols-2 gap-1 my-2"
|
||
></form>
|
||
</details>
|
||
</div>
|
||
<form
|
||
id="form"
|
||
class="flex text-normal px-1 py-1 border border-gray-700 rounded-md items-center"
|
||
>
|
||
<input type="submit" hidden />
|
||
<textarea
|
||
type="text"
|
||
id="prompt"
|
||
class="font-light text-lg w-full px-3 py-2 mx-1 resize-none outline-none"
|
||
oninput="this.style.height = 0;this.style.height = this.scrollHeight + 'px'"
|
||
placeholder="Add your prompt here..."
|
||
>
|
||
Instruct: Write a detailed analogy between mathematics and a lighthouse.
|
||
Output:</textarea
|
||
>
|
||
<button id="clear-btn">
|
||
<svg
|
||
fill="none"
|
||
xmlns="http://www.w3.org/2000/svg"
|
||
width="40"
|
||
viewBox="0 0 70 40"
|
||
>
|
||
<path opacity=".5" d="M39 .2v40.2" stroke="#1F2937" />
|
||
<path
|
||
d="M1.5 11.5 19 29.1m0-17.6L1.5 29.1"
|
||
opacity=".5"
|
||
stroke="#1F2937"
|
||
stroke-width="2"
|
||
/>
|
||
</svg>
|
||
</button>
|
||
<button
|
||
id="run"
|
||
class="bg-gray-700 hover:bg-gray-800 text-white font-normal py-2 w-16 rounded disabled:bg-gray-300 disabled:cursor-not-allowed"
|
||
>
|
||
Run
|
||
</button>
|
||
</form>
|
||
<details>
|
||
<summary class="font-medium cursor-pointer">Advanced Options</summary>
|
||
|
||
<div class="grid grid-cols-3 max-w-md items-center gap-3 py-3">
|
||
<label class="text-sm font-medium" for="max-seq"
|
||
>Maximum length
|
||
</label>
|
||
<input
|
||
type="range"
|
||
id="max-seq"
|
||
name="max-seq"
|
||
min="1"
|
||
max="2048"
|
||
step="1"
|
||
value="200"
|
||
oninput="this.nextElementSibling.value = Number(this.value)"
|
||
/>
|
||
<output
|
||
class="text-xs w-[50px] text-center font-light px-1 py-1 border border-gray-700 rounded-md"
|
||
>
|
||
200</output
|
||
>
|
||
<label class="text-sm font-medium" for="temperature"
|
||
>Temperature</label
|
||
>
|
||
<input
|
||
type="range"
|
||
id="temperature"
|
||
name="temperature"
|
||
min="0"
|
||
max="2"
|
||
step="0.01"
|
||
value="0.00"
|
||
oninput="this.nextElementSibling.value = Number(this.value).toFixed(2)"
|
||
/>
|
||
<output
|
||
class="text-xs w-[50px] text-center font-light px-1 py-1 border border-gray-700 rounded-md"
|
||
>
|
||
0.00</output
|
||
>
|
||
<label class="text-sm font-medium" for="top-p">Top-p</label>
|
||
<input
|
||
type="range"
|
||
id="top-p"
|
||
name="top-p"
|
||
min="0"
|
||
max="1"
|
||
step="0.01"
|
||
value="1.00"
|
||
oninput="this.nextElementSibling.value = Number(this.value).toFixed(2)"
|
||
/>
|
||
<output
|
||
class="text-xs w-[50px] text-center font-light px-1 py-1 border border-gray-700 rounded-md"
|
||
>
|
||
1.00</output
|
||
>
|
||
|
||
<label class="text-sm font-medium" for="repeat_penalty"
|
||
>Repeat Penalty</label
|
||
>
|
||
|
||
<input
|
||
type="range"
|
||
id="repeat_penalty"
|
||
name="repeat_penalty"
|
||
min="1"
|
||
max="2"
|
||
step="0.01"
|
||
value="1.10"
|
||
oninput="this.nextElementSibling.value = Number(this.value).toFixed(2)"
|
||
/>
|
||
<output
|
||
class="text-xs w-[50px] text-center font-light px-1 py-1 border border-gray-700 rounded-md"
|
||
>1.10</output
|
||
>
|
||
<label class="text-sm font-medium" for="seed">Seed</label>
|
||
<input
|
||
type="number"
|
||
id="seed"
|
||
name="seed"
|
||
value="299792458"
|
||
class="font-light border border-gray-700 text-right rounded-md p-2"
|
||
/>
|
||
<button
|
||
id="run"
|
||
onclick="document.querySelector('#seed').value = Math.floor(Math.random() * Number.MAX_SAFE_INTEGER)"
|
||
class="bg-gray-700 hover:bg-gray-800 text-white font-normal py-1 w-[50px] rounded disabled:bg-gray-300 disabled:cursor-not-allowed text-sm"
|
||
>
|
||
Rand
|
||
</button>
|
||
</div>
|
||
</details>
|
||
|
||
<div>
|
||
<h3 class="font-medium">Generation:</h3>
|
||
<div
|
||
class="min-h-[250px] bg-slate-100 text-gray-500 p-4 rounded-md flex flex-col gap-2"
|
||
>
|
||
<div
|
||
id="output-counter"
|
||
hidden
|
||
class="ml-auto font-semibold grid-rows-1"
|
||
></div>
|
||
<p hidden id="output-generation" class="grid-rows-2 text-lg"></p>
|
||
<span id="output-status" class="m-auto font-light"
|
||
>No output yet</span
|
||
>
|
||
</div>
|
||
</div>
|
||
</main>
|
||
</body>
|
||
</html>
|