import init, { tokenize_json } from './pkg/nedotokenizer_space_wasm.js'; import { gzipBytes } from './pkg/nedotokenizer_space_wasm_gzip.js'; const loading = document.querySelector('#loading'); const app = document.querySelector('#app'); const input = document.querySelector('#input'); const tokens = document.querySelector('#tokens'); const count = document.querySelector('#count'); const baseByte = document.querySelector('#base-byte'); const escapeHtml = (value) => String(value) .replaceAll('&', '&') .replaceAll('<', '<') .replaceAll('>', '>') .replaceAll('"', '"') .replaceAll("'", '''); const palette = ['#fde68a', '#bfdbfe', '#bbf7d0', '#fecdd3', '#ddd6fe', '#fed7aa']; function splitGradient(indexes) { if (indexes.length < 2) return ''; const width = 100 / indexes.length; const stops = []; indexes.forEach((tokenIndex, i) => { const color = palette[tokenIndex % palette.length]; const start = (i * width).toFixed(3); const end = ((i + 1) * width).toFixed(3); stops.push(`${color} ${start}%`, `${color} ${end}%`); }); return `background:linear-gradient(90deg,${stops.join(',')})`; } let timer = 0; function render() { const result = JSON.parse(tokenize_json(input.value)); if (!result.ok) { tokens.textContent = 'Tokenization failed.'; count.textContent = '—'; baseByte.textContent = ''; return; } count.textContent = `${result.content_tokens} gerçek 32K token`; baseByte.textContent = result.base_byte_tokens > 0 ? `${result.base_byte_tokens} base-byte token` : 'base-byte token yok'; tokens.innerHTML = result.visual .map((segment) => { const indexes = segment.token_indexes; const ids = segment.token_ids.join(', '); const split = segment.split_utf8; const cls = split ? `t split${segment.base_byte ? ' byte' : ''}` : `t t${indexes[0] % 6}${segment.base_byte ? ' byte' : ''}`; const title = split ? `UTF-8 karakteri token sınırında · IDs ${ids}` : `ID ${ids}`; const style = splitGradient(indexes); return `${escapeHtml(segment.display)}`; }) .join(''); } input.addEventListener('input', () => { clearTimeout(timer); timer = setTimeout(render, 70); }); async function inflateWasm() { if (typeof DecompressionStream !== 'function') { throw new Error('Tarayıcı gzip DecompressionStream desteklemiyor.'); } const compressed = gzipBytes(); const stream = new Blob([compressed]).stream().pipeThrough(new DecompressionStream('gzip')); return await new Response(stream).arrayBuffer(); } try { loading.textContent = 'Tokenizer yükleniyor…'; // Let the loading label paint before decoding/compiling WASM. await new Promise((resolve) => requestAnimationFrame(() => resolve())); const wasm = await inflateWasm(); loading.textContent = 'Tokenizer başlatılıyor…'; await new Promise((resolve) => requestAnimationFrame(() => resolve())); await init(wasm); render(); loading.hidden = true; app.hidden = false; } catch (error) { console.error('NedoTokenizer init failed:', error); const detail = error && error.message ? ` ${error.message}` : ''; loading.textContent = `Yüklenemedi.${detail}`; }