Spaces:
Running
Running
| import init, { tokenize_json } from './pkg/nedotokenizer_space_wasm.js'; | |
| import { gzipBytes } from './pkg/nedotokenizer_space_wasm_gzip.js'; | |
| const loading = document.querySelector('#loading'); | |
| const app = document.querySelector('#app'); | |
| const input = document.querySelector('#input'); | |
| const tokens = document.querySelector('#tokens'); | |
| const count = document.querySelector('#count'); | |
| const baseByte = document.querySelector('#base-byte'); | |
| const escapeHtml = (value) => String(value) | |
| .replaceAll('&', '&') | |
| .replaceAll('<', '<') | |
| .replaceAll('>', '>') | |
| .replaceAll('"', '"') | |
| .replaceAll("'", '''); | |
| const palette = ['#fde68a', '#bfdbfe', '#bbf7d0', '#fecdd3', '#ddd6fe', '#fed7aa']; | |
| function splitGradient(indexes) { | |
| if (indexes.length < 2) return ''; | |
| const width = 100 / indexes.length; | |
| const stops = []; | |
| indexes.forEach((tokenIndex, i) => { | |
| const color = palette[tokenIndex % palette.length]; | |
| const start = (i * width).toFixed(3); | |
| const end = ((i + 1) * width).toFixed(3); | |
| stops.push(`${color} ${start}%`, `${color} ${end}%`); | |
| }); | |
| return `background:linear-gradient(90deg,${stops.join(',')})`; | |
| } | |
| let timer = 0; | |
| function render() { | |
| const result = JSON.parse(tokenize_json(input.value)); | |
| if (!result.ok) { | |
| tokens.textContent = 'Tokenization failed.'; | |
| count.textContent = '—'; | |
| baseByte.textContent = ''; | |
| return; | |
| } | |
| count.textContent = `${result.content_tokens} gerçek 32K token`; | |
| baseByte.textContent = result.base_byte_tokens > 0 | |
| ? `${result.base_byte_tokens} base-byte token` | |
| : 'base-byte token yok'; | |
| tokens.innerHTML = result.visual | |
| .map((segment) => { | |
| const indexes = segment.token_indexes; | |
| const ids = segment.token_ids.join(', '); | |
| const split = segment.split_utf8; | |
| const cls = split | |
| ? `t split${segment.base_byte ? ' byte' : ''}` | |
| : `t t${indexes[0] % 6}${segment.base_byte ? ' byte' : ''}`; | |
| const title = split | |
| ? `UTF-8 karakteri token sınırında · IDs ${ids}` | |
| : `ID ${ids}`; | |
| const style = splitGradient(indexes); | |
| return `<span class="${cls}"${style ? ` style="${style}"` : ''} title="${escapeHtml(title)}">${escapeHtml(segment.display)}</span>`; | |
| }) | |
| .join(''); | |
| } | |
| input.addEventListener('input', () => { | |
| clearTimeout(timer); | |
| timer = setTimeout(render, 70); | |
| }); | |
| async function inflateWasm() { | |
| if (typeof DecompressionStream !== 'function') { | |
| throw new Error('Tarayıcı gzip DecompressionStream desteklemiyor.'); | |
| } | |
| const compressed = gzipBytes(); | |
| const stream = new Blob([compressed]).stream().pipeThrough(new DecompressionStream('gzip')); | |
| return await new Response(stream).arrayBuffer(); | |
| } | |
| try { | |
| loading.textContent = 'Tokenizer yükleniyor…'; | |
| // Let the loading label paint before decoding/compiling WASM. | |
| await new Promise((resolve) => requestAnimationFrame(() => resolve())); | |
| const wasm = await inflateWasm(); | |
| loading.textContent = 'Tokenizer başlatılıyor…'; | |
| await new Promise((resolve) => requestAnimationFrame(() => resolve())); | |
| await init(wasm); | |
| render(); | |
| loading.hidden = true; | |
| app.hidden = false; | |
| } catch (error) { | |
| console.error('NedoTokenizer init failed:', error); | |
| const detail = error && error.message ? ` ${error.message}` : ''; | |
| loading.textContent = `Yüklenemedi.${detail}`; | |
| } | |