Replace localStorage tokenizer cache with IndexedDB

IndexedDB can store large tokenizer JSON blobs without the ~5MB
localStorage cap. Adds async helpers for open/get/set/clear.

script.js

@@ -1,57 +1,114 @@
 import { Tokenizer } from "https://nobsdelivr.private.coffee/npm/@huggingface/tokenizers@0.1.3";
 
-if (localStorage.getItem("tokenizerCache")) {
-  var tokenizerCache = JSON.parse(localStorage.tokenizerCache);
-} else {
-  var tokenizerCache = {};
-  localStorage.tokenizerCache = "{}";
+// IndexedDB helper for caching tokenizer JSON.
+// IndexedDB can store large blobs, unlike localStorage's ~5MB cap.
+const dbName = "tokenizerCacheDB";
+const storeName = "tokenizerCache";
+
+function openDB() {
+  return new Promise((resolve, reject) => {
+    const request = indexedDB.open(dbName, 1);
+
+    request.onupgradeneeded = () => {
+      request.result.createObjectStore(storeName);
+    };
+
+    request.onsuccess = () => resolve(request.result);
+    request.onerror = () => reject(request.error);
+  });
+}
+
+let dbPromise;
+
+async function getCache(modelId) {
+  if (!dbPromise) dbPromise = openDB();
+  const db = await dbPromise;
+
+  return new Promise((resolve, reject) => {
+    const tx = db.transaction(storeName, "readonly");
+    const get = tx.objectStore(storeName).get(modelId);
+
+    get.onsuccess = () => resolve(get.result);
+    get.onerror = () => reject(get.error);
+  });
+}
+
+async function setCache(modelId, tokenizerJSON) {
+  if (!dbPromise) dbPromise = openDB();
+  const db = await dbPromise;
+
+  return new Promise((resolve, reject) => {
+    const tx = db.transaction(storeName, "readwrite");
+    tx.objectStore(storeName).put(tokenizerJSON, modelId);
+
+    tx.oncomplete = () => resolve();
+    tx.onerror = () => reject(tx.error);
+  });
+}
+
+async function clearCache() {
+  if (!dbPromise) dbPromise = openDB();
+  const db = await dbPromise;
+
+  return new Promise((resolve, reject) => {
+    const tx = db.transaction(storeName, "readwrite");
+    tx.objectStore(storeName).clear();
+
+    tx.oncomplete = () => resolve();
+    tx.onerror = () => reject(tx.error);
+  });
 }
-var tokenizerJSON;
 
 let tokenizeButton = document.querySelector("#tokenize");
 tokenizeButton.addEventListener("click", async () => {
   tokenizeButton.disabled = true;
   let modelId = document.querySelector("#model").value;
-  
-  if (modelId in tokenizerCache) {
-    tokenizerJSON = tokenizerCache[modelId];
+
+  let tokenizerJSON;
+  try {
+    tokenizerJSON = await getCache(modelId);
+  } catch (err) {
+    console.error("Failed to read from IndexedDB:", err);
   }
-  else {
+
+  if (tokenizerJSON === undefined) {
     let tokenizerJSONSocket = await fetch(`https://huggingface.co/${modelId}/resolve/main/tokenizer.json`);
     tokenizerJSON = await tokenizerJSONSocket.json();
-    tokenizerCache[modelId] = tokenizerJSON;
-    localStorage.tokenizerCache = JSON.stringify(tokenizerCache);
+    try {
+      await setCache(modelId, tokenizerJSON);
+    } catch (err) {
+      console.error("Failed to write to IndexedDB:", err);
+    }
   }
-  
+
   let tokenizer = new Tokenizer(tokenizerJSON, {});
-  
+
   let text = document.querySelector("#text").value;
   let tokenized = tokenizer.encode(text);
-  
+
   document.querySelector("#totalLength").textContent = `Total length: ${tokenized.ids.length}`;
-  
+
   let output = document.querySelector("#output");
   output.contentDocument.body.innerHTML = "";
-  
+
   let colors = ["#ff00007f", "#00ff007f", "#ffff007f", "#0000ff7f"].sort(() => Math.random() - 0.5);
-  
+
   tokenized.ids.forEach((token, index) => {
     let content = document.createElement("span");
-    
+
     content.textContent = tokenizer.decode([token]);
     content.style.backgroundColor = colors[index % colors.length];
     content.style.fontFamily = "monospace";
     content.style.whiteSpace = "pre-wrap";
-    
+
     content.title = token.toString();
-    
+
     output.contentDocument.body.appendChild(content);
   });
-  
+
   tokenizeButton.disabled = false;
 });
 
 document.querySelector("#clear-cache").addEventListener("click", () => {
-  tokenizerCache = {};
-  localStorage.tokenizerCache = "{}";
+  clearCache().catch((err) => console.error("Failed to clear cache:", err));
 });