| |
| |
| |
|
|
| const fs = require('fs'); |
| const path = require('path'); |
|
|
| |
| const paths = { |
| en: { |
| json: path.resolve(__dirname, '../../../data/demo/public/InfoHighlight-intro.json'), |
| html: path.resolve(__dirname, '../content/home.en.html') |
| }, |
| zh: { |
| json: path.resolve(__dirname, '../../../data/demo/public/CN/InfoHighlight-介绍.json'), |
| html: path.resolve(__dirname, '../content/home.zh.html') |
| } |
| }; |
|
|
| |
| |
| |
|
|
| const TOKEN_SURPRISAL_MAX = 18; |
|
|
| |
| |
| |
| function calculateSurprisal(probability) { |
| return -Math.log2(Math.max(probability, Number.EPSILON)); |
| } |
|
|
| |
| const INTRO_RGB = '255, 71, 64'; |
|
|
| |
| const ALPHA_PRECISION = 2; |
|
|
| |
| |
| |
| function getTokenAlpha(surprisal) { |
| const normalizedValue = surprisal < 0 ? 0 : |
| surprisal >= TOKEN_SURPRISAL_MAX ? 1 : |
| surprisal / TOKEN_SURPRISAL_MAX; |
| const alpha = Math.max(0, Math.min(1, normalizedValue)) * 0.7; |
| return alpha.toFixed(ALPHA_PRECISION); |
| } |
|
|
| |
| |
| |
|
|
| |
| |
| |
| function escapeHtml(text) { |
| return text |
| .replace(/&/g, '&') |
| .replace(/</g, '<') |
| .replace(/>/g, '>') |
| .replace(/"/g, '"') |
| .replace(/'/g, '''); |
| } |
|
|
| |
| |
| |
| function generateColoredHTML(jsonPath) { |
| try { |
| const content = fs.readFileSync(jsonPath, 'utf-8'); |
| const data = JSON.parse(content); |
|
|
| let html = ''; |
| for (const token of data.result.bpe_strings) { |
| const text = token.raw; |
| const prob = token.real_topk[1]; |
| const surprisal = calculateSurprisal(prob); |
| const alpha = getTokenAlpha(surprisal); |
|
|
| const escapedText = escapeHtml(text); |
|
|
| if (text.includes('\n')) { |
| const parts = text.split(/(\n)/); |
| for (const part of parts) { |
| if (part === '\n') { |
| html += '<br>'; |
| } else if (part) { |
| html += `<span class="intro-token" style="--a:${alpha}">${escapeHtml(part)}</span>`; |
| } |
| } |
| } else { |
| html += `<span class="intro-token" style="--a:${alpha}">${escapedText}</span>`; |
| } |
| } |
|
|
| return html; |
| } catch (error) { |
| console.error(`Failed to generate HTML from JSON: ${jsonPath}`, error); |
| return null; |
| } |
| } |
|
|
| |
| |
| |
| |
| function buildIntroHtml(htmlPath, coloredHTML) { |
| try { |
| const original = fs.readFileSync(htmlPath, 'utf-8'); |
|
|
| |
| const regex = /(<div class="intro-brief"[^>]*>)([\s\S]*?)(<\/div>)/; |
|
|
| if (!regex.test(original)) { |
| console.error(`intro-brief not found in ${htmlPath}`); |
| return { ok: false }; |
| } |
|
|
| |
| const replacement = `<div class="intro-brief" style="--intro-rgb: ${INTRO_RGB}">\n ${coloredHTML}\n</div>`; |
| const nextHtml = original.replace(regex, replacement); |
| const changed = nextHtml !== original; |
|
|
| return { ok: true, changed, nextHtml }; |
| } catch (error) { |
| console.error(`Failed to read/update HTML file: ${htmlPath}`, error); |
| return { ok: false }; |
| } |
| } |
|
|
| |
| |
| |
| function main() { |
| const enHTML = generateColoredHTML(paths.en.json); |
| if (!enHTML) { |
| console.error('\n✗ Some generation failed'); |
| process.exit(1); |
| } |
|
|
| const en = buildIntroHtml(paths.en.html, enHTML); |
| if (!en.ok) { |
| console.error('\n✗ Some generation failed'); |
| process.exit(1); |
| } |
|
|
| const zhHTML = generateColoredHTML(paths.zh.json); |
| if (!zhHTML) { |
| console.error('\n✗ Some generation failed'); |
| process.exit(1); |
| } |
|
|
| const zh = buildIntroHtml(paths.zh.html, zhHTML); |
| if (!zh.ok) { |
| console.error('\n✗ Some generation failed'); |
| process.exit(1); |
| } |
|
|
| if (!en.changed && !zh.changed) { |
| return; |
| } |
|
|
| console.log('Generating colored intro HTML from JSON...\n'); |
|
|
| if (en.changed) { |
| fs.writeFileSync(paths.en.html, en.nextHtml, 'utf-8'); |
| console.log(`✓ Updated ${path.basename(paths.en.html)}`); |
| } |
| if (zh.changed) { |
| fs.writeFileSync(paths.zh.html, zh.nextHtml, 'utf-8'); |
| console.log(`✓ Updated ${path.basename(paths.zh.html)}`); |
| } |
|
|
| console.log('\n✓ All intro HTML files generated successfully'); |
| } |
|
|
| |
| main(); |
|
|