Spaces:
Running on Zero
Running on Zero
| """ | |
| PII Reveal - Document Privacy Explorer (v2) | |
| ============================================ | |
| Redesigned frontend addressing ui-critique-1.txt: | |
| - calmer palette, one brand accent, category colors desaturated | |
| - KPI summary cards (with risk level) | |
| - tinted category chips + stacked distribution bar | |
| - premium document viewer with Original / Masked toolbar + focus mode | |
| - inspection-rail sidebar (Filters -> Findings -> Actions) | |
| - hover-linked span <-> sidebar inspection | |
| - unified 8/12/16/24/32 spacing scale | |
| - Inter typography, polished hierarchy | |
| Backend (model, server, endpoints) is identical to app.py. | |
| """ | |
| # ββ stdlib βββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| import dataclasses | |
| import functools | |
| import json | |
| import math | |
| import os | |
| import re | |
| import tempfile | |
| from bisect import bisect_left, bisect_right | |
| from collections.abc import Sequence | |
| from dataclasses import dataclass | |
| from pathlib import Path | |
| from typing import Final | |
| # ββ third-party ββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| import gradio as gr | |
| import spaces | |
| import tiktoken | |
| import torch | |
| import torch.nn.functional as F | |
| from fastapi import UploadFile, File | |
| from fastapi.responses import HTMLResponse, JSONResponse | |
| from huggingface_hub import snapshot_download | |
| from safetensors import safe_open | |
| # ββ configuration ββββββββββββββββββββββββββββββββββββββββββββββββ | |
| MODEL_REPO = os.getenv("MODEL_ID", "charles-first-org/second-model") | |
| HF_TOKEN = os.getenv("HF_TOKEN", None) | |
| MODEL_DIR = Path(snapshot_download(MODEL_REPO, token=HF_TOKEN)) | |
| # Desaturated category palette (~25% lower saturation than v1), paired with | |
| # tint/text tokens so chips, dots, and span highlights stay visually quiet. | |
| CATEGORIES_META = { | |
| "private_person": {"color": "#dc2626", "tint": "rgba(220,38,38,0.08)", "text": "#991b1b", "label": "Person"}, | |
| "private_address": {"color": "#0891b2", "tint": "rgba(8,145,178,0.08)", "text": "#155e75", "label": "Address"}, | |
| "private_email": {"color": "#2563eb", "tint": "rgba(37,99,235,0.08)", "text": "#1e40af", "label": "Email"}, | |
| "private_phone": {"color": "#16a34a", "tint": "rgba(22,163,74,0.08)", "text": "#14532d", "label": "Phone"}, | |
| "private_url": {"color": "#ca8a04", "tint": "rgba(202,138,4,0.10)", "text": "#713f12", "label": "URL"}, | |
| "private_date": {"color": "#9333ea", "tint": "rgba(147,51,234,0.08)", "text": "#6b21a8", "label": "Date"}, | |
| "account_number": {"color": "#ea580c", "tint": "rgba(234,88,12,0.08)", "text": "#7c2d12", "label": "Account"}, | |
| "secret": {"color": "#b91c1c", "tint": "rgba(185,28,28,0.10)", "text": "#7f1d1d", "label": "Secret"}, | |
| } | |
| # ===================================================================== | |
| # MODEL ARCHITECTURE + INFERENCE (from reference implementation) | |
| # ===================================================================== | |
| PRIVACY_FILTER_MODEL_TYPE: Final[str] = "privacy_filter" | |
| REQUIRED_MODEL_CONFIG_KEYS: Final[tuple[str, ...]] = ( | |
| "model_type", "encoding", "num_hidden_layers", "num_experts", | |
| "experts_per_token", "vocab_size", "num_labels", "hidden_size", | |
| "intermediate_size", "head_dim", "num_attention_heads", | |
| "num_key_value_heads", "sliding_window", "bidirectional_context", | |
| "bidirectional_left_context", "bidirectional_right_context", | |
| "default_n_ctx", "initial_context_length", "rope_theta", | |
| "rope_scaling_factor", "rope_ntk_alpha", "rope_ntk_beta", "param_dtype", | |
| ) | |
| BACKGROUND_CLASS_LABEL: Final[str] = "O" | |
| BOUNDARY_PREFIXES: Final[tuple[str, ...]] = ("B", "I", "E", "S") | |
| SPAN_CLASS_NAMES: Final[tuple[str, ...]] = ( | |
| BACKGROUND_CLASS_LABEL, | |
| "account_number", "private_address", "private_date", "private_email", | |
| "private_person", "private_phone", "private_url", "secret", | |
| ) | |
| NER_CLASS_NAMES: Final[tuple[str, ...]] = (BACKGROUND_CLASS_LABEL,) + tuple( | |
| f"{prefix}-{base}" | |
| for base in SPAN_CLASS_NAMES if base != BACKGROUND_CLASS_LABEL | |
| for prefix in BOUNDARY_PREFIXES | |
| ) | |
| VITERBI_TRANSITION_BIAS_KEYS: Final[tuple[str, ...]] = ( | |
| "transition_bias_background_stay", "transition_bias_background_to_start", | |
| "transition_bias_inside_to_continue", "transition_bias_inside_to_end", | |
| "transition_bias_end_to_background", "transition_bias_end_to_start", | |
| ) | |
| DEFAULT_VITERBI_CALIBRATION_PRESET: Final[str] = "default" | |
| def validate_model_config_contract(cfg: dict, *, context: str) -> None: | |
| missing = [k for k in REQUIRED_MODEL_CONFIG_KEYS if k not in cfg] | |
| if missing: | |
| raise ValueError(f"{context} missing keys: {', '.join(missing)}") | |
| if cfg.get("model_type") != PRIVACY_FILTER_MODEL_TYPE: | |
| raise ValueError(f"{context} model_type must be {PRIVACY_FILTER_MODEL_TYPE!r}") | |
| if cfg.get("bidirectional_context") is not True: | |
| raise ValueError(f"{context} must use bidirectional_context=true") | |
| lc, rc = cfg.get("bidirectional_left_context"), cfg.get("bidirectional_right_context") | |
| if not isinstance(lc, int) or not isinstance(rc, int) or lc != rc or lc < 0: | |
| raise ValueError(f"{context} bidirectional context must be equal non-negative ints") | |
| sw = cfg.get("sliding_window") | |
| if sw != 2 * lc + 1: | |
| raise ValueError(f"{context} sliding_window must equal 2*context+1") | |
| if cfg["num_labels"] != 33: | |
| raise ValueError(f"{context} num_labels must be 33") | |
| if cfg["param_dtype"] != "bfloat16": | |
| raise ValueError(f"{context} param_dtype must be bfloat16") | |
| # ββ model helpers ββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def expert_linear(x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor | None) -> torch.Tensor: | |
| n, e, k = x.shape | |
| _, _, _, o = weight.shape | |
| out = torch.bmm(x.reshape(n * e, 1, k), weight.reshape(n * e, k, o)).reshape(n, e, o) | |
| return out + bias if bias is not None else out | |
| class ModelConfig: | |
| num_hidden_layers: int; num_experts: int; experts_per_token: int | |
| vocab_size: int; num_labels: int; hidden_size: int; intermediate_size: int | |
| head_dim: int; num_attention_heads: int; num_key_value_heads: int | |
| bidirectional_context_size: int; initial_context_length: int | |
| rope_theta: float; rope_scaling_factor: float; rope_ntk_alpha: float; rope_ntk_beta: float | |
| def from_checkpoint_config(cls, cfg: dict, *, context: str) -> "ModelConfig": | |
| cfg = dict(cfg) | |
| cfg["bidirectional_context_size"] = cfg["bidirectional_left_context"] | |
| fields = {f.name for f in dataclasses.fields(cls)} | |
| return cls(**{k: v for k, v in cfg.items() if k in fields}) | |
| class RMSNorm(torch.nn.Module): | |
| def __init__(self, n: int, eps: float = 1e-5, device=None): | |
| super().__init__() | |
| self.eps = eps | |
| self.scale = torch.nn.Parameter(torch.ones(n, device=device, dtype=torch.float32)) | |
| def forward(self, x): | |
| t = x.float() | |
| return (t * torch.rsqrt(t.pow(2).mean(-1, keepdim=True) + self.eps) * self.scale).to(x.dtype) | |
| def apply_rope(x, cos, sin): | |
| cos = cos.unsqueeze(-2).to(x.dtype); sin = sin.unsqueeze(-2).to(x.dtype) | |
| x1, x2 = x[..., ::2], x[..., 1::2] | |
| return torch.stack((x1 * cos - x2 * sin, x2 * cos + x1 * sin), dim=-1).reshape(x.shape) | |
| class RotaryEmbedding(torch.nn.Module): | |
| def __init__(self, head_dim, base, dtype, *, initial_context_length=4096, | |
| scaling_factor=1.0, ntk_alpha=1.0, ntk_beta=32.0, device=None): | |
| super().__init__() | |
| self.head_dim, self.base, self.dtype = head_dim, base, dtype | |
| self.initial_context_length = initial_context_length | |
| self.scaling_factor, self.ntk_alpha, self.ntk_beta = scaling_factor, ntk_alpha, ntk_beta | |
| self.device = device | |
| mp = max(int(initial_context_length * scaling_factor), initial_context_length) | |
| self.max_position_embeddings = mp | |
| cos, sin = self._compute(mp, device=torch.device("cpu")) | |
| target = device or torch.device("cpu") | |
| self.register_buffer("cos_cache", cos.to(target), persistent=False) | |
| self.register_buffer("sin_cache", sin.to(target), persistent=False) | |
| def _inv_freq(self, device=None): | |
| device = device or self.device | |
| freq = self.base ** (torch.arange(0, self.head_dim, 2, dtype=torch.float, device=device) / self.head_dim) | |
| if self.scaling_factor > 1.0: | |
| d_half = self.head_dim / 2 | |
| low = d_half * math.log(self.initial_context_length / (self.ntk_beta * 2 * math.pi)) / math.log(self.base) | |
| high = d_half * math.log(self.initial_context_length / (self.ntk_alpha * 2 * math.pi)) / math.log(self.base) | |
| interp = 1.0 / (self.scaling_factor * freq) | |
| extrap = 1.0 / freq | |
| ramp = (torch.arange(d_half, dtype=torch.float32, device=device) - low) / (high - low) | |
| mask = 1 - ramp.clamp(0, 1) | |
| return interp * (1 - mask) + extrap * mask | |
| return 1.0 / freq | |
| def _compute(self, n, device=None): | |
| inv_freq = self._inv_freq(device) | |
| t = torch.arange(n, dtype=torch.float32, device=device or self.device) | |
| freqs = torch.einsum("i,j->ij", t, inv_freq) | |
| c = 0.1 * math.log(self.scaling_factor) + 1.0 if self.scaling_factor > 1.0 else 1.0 | |
| return (freqs.cos() * c).to(self.dtype), (freqs.sin() * c).to(self.dtype) | |
| def forward(self, q, k): | |
| n = q.shape[0] | |
| if n > self.cos_cache.shape[0]: | |
| cos, sin = self._compute(n, torch.device("cpu")) | |
| self.cos_cache, self.sin_cache = cos.to(q.device), sin.to(q.device) | |
| cc = self.cos_cache.to(q.device) if self.cos_cache.device != q.device else self.cos_cache | |
| sc = self.sin_cache.to(q.device) if self.sin_cache.device != q.device else self.sin_cache | |
| cos, sin = cc[:n], sc[:n] | |
| q = apply_rope(q.view(n, -1, self.head_dim), cos, sin).reshape(q.shape) | |
| k = apply_rope(k.view(n, -1, self.head_dim), cos, sin).reshape(k.shape) | |
| return q, k | |
| def sdpa(Q, K, V, S, sm_scale, ctx): | |
| n, nh, qm, hd = Q.shape | |
| w = 2 * ctx + 1 | |
| Kp = F.pad(K, (0, 0, 0, 0, ctx, ctx)); Vp = F.pad(V, (0, 0, 0, 0, ctx, ctx)) | |
| Kw = Kp.unfold(0, w, 1).permute(0, 3, 1, 2); Vw = Vp.unfold(0, w, 1).permute(0, 3, 1, 2) | |
| idx = torch.arange(w, device=Q.device) - ctx | |
| pos = torch.arange(n, device=Q.device)[:, None] + idx[None, :] | |
| valid = (pos >= 0) & (pos < n) | |
| scores = torch.einsum("nhqd,nwhd->nhqw", Q, Kw).float() * sm_scale | |
| scores = scores.masked_fill(~valid[:, None, None, :], -float("inf")) | |
| sink = (S * math.log(2.0)).reshape(nh, qm)[None, :, :, None].expand(n, -1, -1, 1) | |
| scores = torch.cat([scores, sink], dim=-1) | |
| wt = torch.softmax(scores, dim=-1)[..., :-1].to(V.dtype) | |
| return torch.einsum("nhqw,nwhd->nhqd", wt, Vw).reshape(n, -1) | |
| class AttentionBlock(torch.nn.Module): | |
| def __init__(self, cfg: ModelConfig, device=None): | |
| super().__init__() | |
| dt = torch.bfloat16 | |
| self.head_dim, self.nah, self.nkv = cfg.head_dim, cfg.num_attention_heads, cfg.num_key_value_heads | |
| self.ctx = int(cfg.bidirectional_context_size) | |
| self.sinks = torch.nn.Parameter(torch.empty(cfg.num_attention_heads, device=device, dtype=torch.float32)) | |
| self.norm = RMSNorm(cfg.hidden_size, device=device) | |
| qkv_d = cfg.head_dim * (cfg.num_attention_heads + 2 * cfg.num_key_value_heads) | |
| self.qkv = torch.nn.Linear(cfg.hidden_size, qkv_d, device=device, dtype=dt) | |
| self.out = torch.nn.Linear(cfg.head_dim * cfg.num_attention_heads, cfg.hidden_size, device=device, dtype=dt) | |
| self.qk_scale = 1 / math.sqrt(math.sqrt(cfg.head_dim)) | |
| self.rope = RotaryEmbedding(cfg.head_dim, int(cfg.rope_theta), torch.float32, | |
| initial_context_length=cfg.initial_context_length, | |
| scaling_factor=cfg.rope_scaling_factor, | |
| ntk_alpha=cfg.rope_ntk_alpha, ntk_beta=cfg.rope_ntk_beta, device=device) | |
| def forward(self, x): | |
| t = self.norm(x).to(self.qkv.weight.dtype) | |
| qkv = F.linear(t, self.qkv.weight, self.qkv.bias) | |
| hd, nah, nkv = self.head_dim, self.nah, self.nkv | |
| q = qkv[:, :nah * hd].contiguous() | |
| k = qkv[:, nah * hd:(nah + nkv) * hd].contiguous() | |
| v = qkv[:, (nah + nkv) * hd:(nah + 2 * nkv) * hd].contiguous() | |
| q, k = self.rope(q, k) | |
| q, k = q * self.qk_scale, k * self.qk_scale | |
| n = q.shape[0] | |
| q = q.view(n, nkv, nah // nkv, hd); k = k.view(n, nkv, hd); v = v.view(n, nkv, hd) | |
| ao = sdpa(q, k, v, self.sinks, 1.0, self.ctx).to(self.out.weight.dtype) | |
| return x + F.linear(ao, self.out.weight, self.out.bias).to(x.dtype) | |
| def swiglu(x, alpha=1.702, limit=7.0): | |
| g, l = x.chunk(2, dim=-1) | |
| g, l = g.clamp(max=limit), l.clamp(-limit, limit) | |
| return g * torch.sigmoid(alpha * g) * (l + 1) | |
| class MLPBlock(torch.nn.Module): | |
| def __init__(self, cfg: ModelConfig, device=None): | |
| super().__init__() | |
| dt = torch.bfloat16 | |
| self.ne, self.ept = cfg.num_experts, cfg.experts_per_token | |
| self.norm = RMSNorm(cfg.hidden_size, device=device) | |
| self.gate = torch.nn.Linear(cfg.hidden_size, cfg.num_experts, device=device, dtype=dt) | |
| self.mlp1_weight = torch.nn.Parameter(torch.empty(cfg.num_experts, cfg.hidden_size, cfg.intermediate_size * 2, device=device, dtype=dt)) | |
| self.mlp1_bias = torch.nn.Parameter(torch.empty(cfg.num_experts, cfg.intermediate_size * 2, device=device, dtype=dt)) | |
| self.mlp2_weight = torch.nn.Parameter(torch.empty(cfg.num_experts, cfg.intermediate_size, cfg.hidden_size, device=device, dtype=dt)) | |
| self.mlp2_bias = torch.nn.Parameter(torch.empty(cfg.num_experts, cfg.hidden_size, device=device, dtype=dt)) | |
| def forward(self, x): | |
| t = self.norm(x) | |
| gs = F.linear(t.float(), self.gate.weight.float(), self.gate.bias.float()) | |
| top = torch.topk(gs, k=self.ept, dim=-1, sorted=True) | |
| ew = torch.softmax(top.values, dim=-1) / self.ept | |
| ei = top.indices | |
| ept = self.ept | |
| def _chunk(tc, eic, ewc): | |
| o = expert_linear(tc.float().unsqueeze(1).expand(-1, eic.shape[1], -1), | |
| self.mlp1_weight[eic].float(), self.mlp1_bias[eic].float()) | |
| o = swiglu(o) | |
| o = expert_linear(o.float(), self.mlp2_weight[eic].float(), self.mlp2_bias[eic].float()) | |
| return (torch.einsum("bec,be->bc", o.to(ewc.dtype), ewc) * ept).to(x.dtype) | |
| cs = 32 | |
| if t.shape[0] > cs: | |
| parts = [_chunk(t[s:s+cs], ei[s:s+cs], ew[s:s+cs]) for s in range(0, t.shape[0], cs)] | |
| return x + torch.cat(parts, 0) | |
| return x + _chunk(t, ei, ew) | |
| class TransformerBlock(torch.nn.Module): | |
| def __init__(self, cfg, device=None): | |
| super().__init__() | |
| self.attn = AttentionBlock(cfg, device=device) | |
| self.mlp = MLPBlock(cfg, device=device) | |
| def forward(self, x): | |
| return self.mlp(self.attn(x)) | |
| class Checkpoint: | |
| def build_param_name_map(n): | |
| return ({f"block.{i}.mlp.mlp1_bias": f"block.{i}.mlp.swiglu.bias" for i in range(n)} | |
| | {f"block.{i}.mlp.mlp1_weight": f"block.{i}.mlp.swiglu.weight" for i in range(n)} | |
| | {f"block.{i}.mlp.mlp2_bias": f"block.{i}.mlp.out.bias" for i in range(n)} | |
| | {f"block.{i}.mlp.mlp2_weight": f"block.{i}.mlp.out.weight" for i in range(n)}) | |
| def __init__(self, path, device, num_hidden_layers): | |
| self.pnm = self.build_param_name_map(num_hidden_layers) | |
| self.ds = device.type if device.index is None else f"{device.type}:{device.index}" | |
| files = [os.path.join(path, f) for f in os.listdir(path) if f.endswith(".safetensors")] | |
| self.map = {} | |
| for sf in files: | |
| with safe_open(sf, framework="pt", device=self.ds) as h: | |
| for k in h.keys(): | |
| self.map[k] = sf | |
| def get(self, name): | |
| mapped = self.pnm.get(name, name) | |
| with safe_open(self.map[mapped], framework="pt", device=self.ds) as h: | |
| return h.get_tensor(mapped) | |
| class Transformer(torch.nn.Module): | |
| def __init__(self, cfg, device): | |
| super().__init__() | |
| dt = torch.bfloat16 | |
| self.embedding = torch.nn.Embedding(cfg.vocab_size, cfg.hidden_size, device=device, dtype=dt) | |
| self.block = torch.nn.ModuleList([TransformerBlock(cfg, device=device) for _ in range(cfg.num_hidden_layers)]) | |
| self.norm = RMSNorm(cfg.hidden_size, device=device) | |
| self.unembedding = torch.nn.Linear(cfg.hidden_size, cfg.num_labels, bias=False, device=device, dtype=dt) | |
| def forward(self, token_ids): | |
| x = self.embedding(token_ids) | |
| for blk in self.block: | |
| x = blk(x) | |
| return F.linear(self.norm(x), self.unembedding.weight, None) | |
| def from_checkpoint(cls, checkpoint_dir, *, device): | |
| torch.backends.cuda.matmul.allow_tf32 = False | |
| torch.backends.cudnn.allow_tf32 = False | |
| torch.set_float32_matmul_precision("highest") | |
| cp = json.loads((Path(checkpoint_dir) / "config.json").read_text()) | |
| validate_model_config_contract(cp, context=str(checkpoint_dir)) | |
| cfg = ModelConfig.from_checkpoint_config(cp, context=str(checkpoint_dir)) | |
| ckpt = Checkpoint(checkpoint_dir, device, cfg.num_hidden_layers) | |
| m = cls(cfg, device); m.eval() | |
| for name, param in m.named_parameters(): | |
| loaded = ckpt.get(name) | |
| if param.shape != loaded.shape: | |
| raise ValueError(f"Shape mismatch {name}: {param.shape} vs {loaded.shape}") | |
| param.data.copy_(loaded) | |
| return m | |
| # ββ label info + span decoding βββββββββββββββββββββββββββββββββββ | |
| class LabelInfo: | |
| boundary_label_lookup: dict[str, dict[str, int]] | |
| token_to_span_label: dict[int, int] | |
| token_boundary_tags: dict[int, str | None] | |
| span_class_names: tuple[str, ...] | |
| span_label_lookup: dict[str, int] | |
| background_token_label: int | |
| background_span_label: int | |
| def labels_to_spans(labels_by_index, label_info): | |
| spans, cur_label, start_idx, prev_idx = [], None, None, None | |
| bg = label_info.background_span_label | |
| for ti in sorted(labels_by_index): | |
| lid = labels_by_index[ti] | |
| sl = label_info.token_to_span_label.get(lid) | |
| bt = label_info.token_boundary_tags.get(lid) | |
| if prev_idx is not None and ti != prev_idx + 1: | |
| if cur_label is not None and start_idx is not None: | |
| spans.append((cur_label, start_idx, prev_idx + 1)) | |
| cur_label = start_idx = None | |
| if sl is None: | |
| prev_idx = ti; continue | |
| if sl == bg: | |
| if cur_label is not None and start_idx is not None: | |
| spans.append((cur_label, start_idx, ti)) | |
| cur_label = start_idx = None; prev_idx = ti; continue | |
| if bt == "S": | |
| if cur_label is not None and start_idx is not None and prev_idx is not None: | |
| spans.append((cur_label, start_idx, prev_idx + 1)) | |
| spans.append((sl, ti, ti + 1)); cur_label = start_idx = None | |
| elif bt == "B": | |
| if cur_label is not None and start_idx is not None and prev_idx is not None: | |
| spans.append((cur_label, start_idx, prev_idx + 1)) | |
| cur_label, start_idx = sl, ti | |
| elif bt == "I": | |
| if cur_label is None or cur_label != sl: | |
| if cur_label is not None and start_idx is not None and prev_idx is not None: | |
| spans.append((cur_label, start_idx, prev_idx + 1)) | |
| cur_label, start_idx = sl, ti | |
| elif bt == "E": | |
| if cur_label is None or cur_label != sl or start_idx is None: | |
| if cur_label is not None and start_idx is not None and prev_idx is not None: | |
| spans.append((cur_label, start_idx, prev_idx + 1)) | |
| spans.append((sl, ti, ti + 1)); cur_label = start_idx = None | |
| else: | |
| spans.append((cur_label, start_idx, ti + 1)); cur_label = start_idx = None | |
| else: | |
| if cur_label is not None and start_idx is not None and prev_idx is not None: | |
| spans.append((cur_label, start_idx, prev_idx + 1)) | |
| cur_label = start_idx = None | |
| prev_idx = ti | |
| if cur_label is not None and start_idx is not None and prev_idx is not None: | |
| spans.append((cur_label, start_idx, prev_idx + 1)) | |
| return spans | |
| def token_spans_to_char_spans(spans, cs, ce): | |
| out = [] | |
| for li, ts, te in spans: | |
| if not (0 <= ts < te <= len(cs)): | |
| continue | |
| s, e = cs[ts], ce[te - 1] | |
| if e > s: | |
| out.append((li, s, e)) | |
| return out | |
| def trim_char_spans_whitespace(spans, text): | |
| out = [] | |
| for li, s, e in spans: | |
| if not (0 <= s < e <= len(text)): | |
| continue | |
| while s < e and text[s].isspace(): s += 1 | |
| while e > s and text[e - 1].isspace(): e -= 1 | |
| if e > s: | |
| out.append((li, s, e)) | |
| return out | |
| # ββ viterbi decoder ββββββββββββββββββββββββββββββββββββββββββββββ | |
| def get_viterbi_transition_biases(): | |
| cp = MODEL_DIR / "viterbi_calibration.json" | |
| default = {k: 0.0 for k in VITERBI_TRANSITION_BIAS_KEYS} | |
| if not cp.is_file(): | |
| return default | |
| payload = json.loads(cp.read_text()) | |
| raw = payload | |
| ops = payload.get("operating_points") | |
| if isinstance(ops, dict): | |
| preset = ops.get(DEFAULT_VITERBI_CALIBRATION_PRESET) | |
| if isinstance(preset, dict): | |
| raw = preset.get("biases", raw) | |
| if not isinstance(raw, dict): | |
| return default | |
| return {k: float(raw.get(k, 0.0)) for k in VITERBI_TRANSITION_BIAS_KEYS} | |
| class Decoder: | |
| def __init__(self, label_info): | |
| nc = len(label_info.token_to_span_label) | |
| self._start = torch.full((nc,), -1e9, dtype=torch.float32) | |
| self._end = torch.full((nc,), -1e9, dtype=torch.float32) | |
| self._trans = torch.full((nc, nc), -1e9, dtype=torch.float32) | |
| biases = get_viterbi_transition_biases() | |
| bg_tok, bg_sp = label_info.background_token_label, label_info.background_span_label | |
| ttsl, tbt = label_info.token_to_span_label, label_info.token_boundary_tags | |
| for i in range(nc): | |
| tag, sl = tbt.get(i), ttsl.get(i) | |
| if tag in {"B", "S"} or i == bg_tok: self._start[i] = 0.0 | |
| if tag in {"E", "S"} or i == bg_tok: self._end[i] = 0.0 | |
| for j in range(nc): | |
| nt, ns = tbt.get(j), ttsl.get(j) | |
| if self._valid(tag, sl, nt, ns, bg_tok, bg_sp, j): | |
| self._trans[i, j] = self._bias(tag, sl, nt, ns, bg_sp, biases) | |
| def _valid(pt, ps, nt, ns, bti, bsi, ni): | |
| nb = ns == bsi or ni == bti | |
| if (ns is None or nt is None) and not nb: return False | |
| if pt is None or ps is None: return nb or nt in {"B", "S"} | |
| if ps == bsi or pt in {"E", "S"}: return nb or nt in {"B", "S"} | |
| if pt in {"B", "I"}: return ps == ns and nt in {"I", "E"} | |
| return False | |
| def _bias(pt, ps, nt, ns, bsi, b): | |
| nb, pb = ns == bsi, ps == bsi | |
| if pb: return b["transition_bias_background_stay"] if nb else b["transition_bias_background_to_start"] | |
| if pt in {"B", "I"}: return b["transition_bias_inside_to_continue"] if nt == "I" else b["transition_bias_inside_to_end"] | |
| return b["transition_bias_end_to_background"] if nb else b["transition_bias_end_to_start"] | |
| def decode(self, lp): | |
| sl, nc = lp.shape | |
| if sl == 0: return [] | |
| st = self._start.to(lp.device, lp.dtype) | |
| en = self._end.to(lp.device, lp.dtype) | |
| tr = self._trans.to(lp.device, lp.dtype) | |
| scores = lp[0] + st | |
| bp = torch.empty((sl - 1, nc), device=lp.device, dtype=torch.int64) | |
| for i in range(1, sl): | |
| t = scores.unsqueeze(1) + tr | |
| bs, bi = t.max(dim=0) | |
| scores = bs + lp[i]; bp[i - 1] = bi | |
| if not torch.isfinite(scores).any(): return lp.argmax(dim=1).tolist() | |
| scores += en | |
| path = torch.empty(sl, device=lp.device, dtype=torch.int64) | |
| path[-1] = scores.argmax() | |
| for i in range(sl - 2, -1, -1): path[i] = bp[i, path[i + 1]] | |
| return path.tolist() | |
| # ββ runtime singleton ββββββββββββββββββββββββββββββββββββββββββββ | |
| class InferenceRuntime: | |
| model: Transformer; encoding: tiktoken.Encoding; label_info: LabelInfo | |
| device: torch.device; n_ctx: int | |
| def get_runtime(): | |
| cp = MODEL_DIR | |
| cfg = json.loads((cp / "config.json").read_text()) | |
| validate_model_config_contract(cfg, context=str(cp)) | |
| device = torch.device("cuda") | |
| encoding = tiktoken.get_encoding(str(cfg["encoding"]).strip()) | |
| scn = [BACKGROUND_CLASS_LABEL]; sll = {BACKGROUND_CLASS_LABEL: 0} | |
| bll, ttsl, tbt = {}, {}, {} | |
| bg_idx = None | |
| for idx, name in enumerate(NER_CLASS_NAMES): | |
| if name == BACKGROUND_CLASS_LABEL: | |
| bg_idx = idx; ttsl[idx] = 0; tbt[idx] = None; continue | |
| bnd, base = name.split("-", 1) | |
| si = sll.get(base) | |
| if si is None: | |
| si = len(scn); scn.append(base); sll[base] = si | |
| ttsl[idx] = si; tbt[idx] = bnd | |
| bll.setdefault(base, {})[bnd] = idx | |
| li = LabelInfo(bll, ttsl, tbt, tuple(scn), sll, bg_idx, 0) | |
| m = Transformer.from_checkpoint(str(cp), device=device) | |
| return InferenceRuntime(m, encoding, li, device, int(cfg["default_n_ctx"])) | |
| def predict_text(runtime, text, decoder): | |
| tids = tuple(int(t) for t in runtime.encoding.encode(text, allowed_special="all")) | |
| if not tids: return text, [] | |
| scores = [] | |
| for s in range(0, len(tids), runtime.n_ctx): | |
| e = min(s + runtime.n_ctx, len(tids)) | |
| wt = torch.tensor(tids[s:e], device=runtime.device, dtype=torch.int32) | |
| lp = F.log_softmax(runtime.model(wt).float(), dim=-1) | |
| scores.extend(lp.unbind(0)) | |
| stacked = torch.stack(scores, 0) | |
| dl = decoder.decode(stacked) | |
| if len(dl) != len(tids): dl = stacked.argmax(dim=1).tolist() | |
| pli = {i: int(l) for i, l in enumerate(dl)} | |
| pts = labels_to_spans(pli, runtime.label_info) | |
| tb = [runtime.encoding.decode_single_token_bytes(t) for t in tids] | |
| dt = b"".join(tb).decode("utf-8", errors="replace") | |
| cbs, cbe = [], [] | |
| bc = 0 | |
| for ch in dt: cbs.append(bc); bc += len(ch.encode("utf-8")); cbe.append(bc) | |
| cs, ce = [], [] | |
| tbc = 0 | |
| for rb in tb: | |
| tbs = tbc; tbe = tbs + len(rb); tbc = tbe | |
| cs.append(bisect_right(cbe, tbs)); ce.append(bisect_left(cbs, tbe)) | |
| pcs = token_spans_to_char_spans(pts, cs, ce) | |
| pcs = trim_char_spans_whitespace(pcs, dt if dt != text else text) | |
| src = dt if dt != text else text | |
| detected = [] | |
| for li, s, e in pcs: | |
| if 0 <= li < len(runtime.label_info.span_class_names): | |
| lbl = runtime.label_info.span_class_names[li] | |
| else: | |
| lbl = f"label_{li}" | |
| detected.append({"label": lbl, "start": s, "end": e, "text": src[s:e]}) | |
| return src, detected | |
| # ===================================================================== | |
| # APPLICATION LAYER | |
| # ===================================================================== | |
| def extract_text(file_path: str) -> str: | |
| suffix = Path(file_path).suffix.lower() | |
| if suffix == ".pdf": | |
| import fitz | |
| doc = fitz.open(file_path) | |
| pages = [page.get_text() for page in doc] | |
| doc.close() | |
| return "\n\n".join(pages) | |
| elif suffix in (".docx", ".doc"): | |
| from docx import Document | |
| doc = Document(file_path) | |
| return "\n\n".join(p.text for p in doc.paragraphs if p.text.strip()) | |
| raise ValueError(f"Unsupported file type: {suffix}") | |
| def compute_stats(text, spans): | |
| total = len(text) | |
| pii_chars = sum(s["end"] - s["start"] for s in spans) | |
| by_cat = {} | |
| for s in spans: | |
| c = s["label"] | |
| by_cat.setdefault(c, {"count": 0, "chars": 0}) | |
| by_cat[c]["count"] += 1; by_cat[c]["chars"] += s["end"] - s["start"] | |
| pct = round(pii_chars / total * 100, 1) if total else 0 | |
| # Risk tiering β secrets/accounts/emails make a document higher-risk even | |
| # at low coverage, so combine a density rule with a sensitive-category rule. | |
| sensitive = sum(by_cat.get(k, {}).get("count", 0) for k in ("secret", "account_number", "private_email")) | |
| if pct >= 15 or sensitive >= 5: | |
| risk = "High" | |
| elif pct >= 5 or sensitive >= 1 or len(spans) >= 10: | |
| risk = "Medium" | |
| else: | |
| risk = "Low" | |
| return { | |
| "total_chars": total, "pii_chars": pii_chars, | |
| "pii_percentage": pct, | |
| "total_spans": len(spans), "categories": by_cat, "num_categories": len(by_cat), | |
| "risk_level": risk, | |
| } | |
| def detect_speakers(text, spans): | |
| patterns = [r"^([A-Z][a-zA-Z ]{1,30}):\s", r"^\[([^\]]{1,30})\]\s", r"^(Speaker\s*\d+):\s"] | |
| line_sp, pos, cur = [], 0, None | |
| for line in text.split("\n"): | |
| for p in patterns: | |
| m = re.match(p, line) | |
| if m: cur = m.group(1).strip(); break | |
| line_sp.append((pos, pos + len(line), cur)); pos += len(line) + 1 | |
| result = {} | |
| for span in spans: | |
| mid = (span["start"] + span["end"]) // 2 | |
| speaker = "Document" | |
| for ls, le, sp in line_sp: | |
| if ls <= mid <= le and sp: speaker = sp; break | |
| result[speaker] = result.get(speaker, 0) + 1 | |
| return {} if list(result.keys()) == ["Document"] else result | |
| def run_pii_analysis(text: str): | |
| """GPU-accelerated PII detection.""" | |
| runtime = get_runtime() | |
| decoder = Decoder(label_info=runtime.label_info) | |
| source_text, detected = predict_text(runtime, text, decoder) | |
| return source_text, detected | |
| # ββ Gradio Server ββββββββββββββββββββββββββββββββββββββββββββββββ | |
| server = gr.Server() | |
| async def homepage(): | |
| return FRONTEND_HTML | |
| async def analyze_document(file: UploadFile = File(...)): | |
| suffix = Path(file.filename).suffix.lower() | |
| if suffix not in (".pdf", ".doc", ".docx"): | |
| return JSONResponse({"error": f"Unsupported: {suffix}. Use PDF, DOC, or DOCX."}, 400) | |
| with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp: | |
| tmp.write(await file.read()); tmp_path = tmp.name | |
| try: | |
| text = extract_text(tmp_path) | |
| if not text.strip(): | |
| return JSONResponse({"error": "No text content found."}, 400) | |
| source_text, spans = run_pii_analysis(text) | |
| stats = compute_stats(source_text, spans) | |
| speakers = detect_speakers(source_text, spans) | |
| return JSONResponse({ | |
| "filename": file.filename, "text": source_text, "spans": spans, | |
| "stats": stats, "speakers": speakers, | |
| "categories_meta": {k: {"color": v["color"], "tint": v["tint"], "text": v["text"], "label": v["label"]} | |
| for k, v in CATEGORIES_META.items()}, | |
| }) | |
| except Exception as e: | |
| return JSONResponse({"error": str(e)}, 500) | |
| finally: | |
| if os.path.exists(tmp_path): os.unlink(tmp_path) | |
| def analyze_text_api(text: str) -> str: | |
| """Gradio API: analyze raw text for PII.""" | |
| source_text, spans = run_pii_analysis(text) | |
| stats = compute_stats(source_text, spans) | |
| return json.dumps({"text": source_text, "spans": spans, "stats": stats}, ensure_ascii=False) | |
| # ββ Frontend HTML (redesigned) βββββββββββββββββββββββββββββββββββ | |
| FRONTEND_HTML = r"""<!DOCTYPE html> | |
| <html lang="en"> | |
| <head> | |
| <meta charset="UTF-8"> | |
| <meta name="viewport" content="width=device-width,initial-scale=1"> | |
| <title>PII Reveal</title> | |
| <link rel="preconnect" href="https://fonts.googleapis.com"> | |
| <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin> | |
| <link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700;800&family=JetBrains+Mono:wght@400;500&display=swap" rel="stylesheet"> | |
| <style> | |
| *,*::before,*::after{box-sizing:border-box;margin:0;padding:0} | |
| :root{ | |
| /* Neutral base */ | |
| --bg: #f7f7f9; | |
| --surface: #ffffff; | |
| --surface-2: #fafbfc; | |
| --surface-warm: #fdfdfb; | |
| --border: #e4e7ec; | |
| --border-soft: #eef0f3; | |
| --text: #0f172a; | |
| --text-2: #475569; | |
| --text-3: #94a3b8; | |
| /* Brand accent β one gradient, used sparingly */ | |
| --brand: #7c3aed; | |
| --brand-2: #ec4899; | |
| --brand-soft: rgba(124,58,237,.08); | |
| --brand-ring: rgba(124,58,237,.22); | |
| /* Risk */ | |
| --risk-high: #b91c1c; | |
| --risk-med: #b45309; | |
| --risk-low: #15803d; | |
| /* Spacing scale β 8 / 12 / 16 / 24 / 32 / 48 */ | |
| --s-1: 8px; --s-2: 12px; --s-3: 16px; --s-4: 24px; --s-5: 32px; --s-6: 48px; | |
| /* Radius */ | |
| --r-sm: 8px; --r-md: 12px; --r-lg: 16px; --r-xl: 20px; | |
| /* Shadow β subtle */ | |
| --shadow-xs: 0 1px 2px rgba(15,23,42,.04); | |
| --shadow-sm: 0 1px 3px rgba(15,23,42,.06), 0 1px 2px rgba(15,23,42,.04); | |
| --shadow-md: 0 4px 16px rgba(15,23,42,.06), 0 2px 4px rgba(15,23,42,.04); | |
| --shadow-lg: 0 12px 40px rgba(15,23,42,.10); | |
| } | |
| html,body{height:100%} | |
| body{ | |
| font-family:'Inter',system-ui,-apple-system,sans-serif; | |
| background:var(--bg); | |
| color:var(--text); | |
| font-feature-settings:"cv11","ss01","ss03"; | |
| font-size:15px; | |
| line-height:1.5; | |
| -webkit-font-smoothing:antialiased; | |
| } | |
| button{font-family:inherit} | |
| /* =============== UPLOAD VIEW =============== */ | |
| #upload-view{ | |
| display:flex;align-items:center;justify-content:center; | |
| min-height:100vh;padding:var(--s-5); | |
| } | |
| .upload-card{ | |
| background:var(--surface); | |
| border:1px solid var(--border); | |
| border-radius:var(--r-xl); | |
| padding:var(--s-6) var(--s-5); | |
| max-width:600px;width:100%; | |
| box-shadow:var(--shadow-lg); | |
| text-align:center; | |
| } | |
| .brand{display:inline-flex;align-items:center;gap:var(--s-2)} | |
| .brand-logo{ | |
| width:36px;height:36px;border-radius:10px; | |
| background:linear-gradient(135deg,var(--brand) 0%,var(--brand-2) 100%); | |
| display:flex;align-items:center;justify-content:center; | |
| color:#fff;font-weight:700;font-size:16px; | |
| box-shadow:0 4px 12px rgba(124,58,237,.25); | |
| } | |
| .brand-name{ | |
| font-size:18px;font-weight:700;letter-spacing:-.01em; | |
| background:linear-gradient(135deg,var(--brand),var(--brand-2)); | |
| -webkit-background-clip:text;-webkit-text-fill-color:transparent; | |
| } | |
| .upload-card .brand{margin-bottom:var(--s-2)} | |
| .upload-hero{ | |
| font-size:28px;font-weight:700;letter-spacing:-.02em; | |
| margin-top:var(--s-2); | |
| } | |
| .upload-sub{color:var(--text-2);margin-top:var(--s-1);font-size:15px} | |
| .dropzone{ | |
| margin-top:var(--s-5); | |
| border:1.5px dashed var(--border); | |
| background:var(--surface-2); | |
| border-radius:var(--r-lg); | |
| padding:var(--s-5) var(--s-4); | |
| cursor:pointer;transition:all .18s; | |
| position:relative; | |
| } | |
| .dropzone:hover,.dropzone.dragover{ | |
| border-color:var(--brand);background:var(--brand-soft); | |
| } | |
| .dropzone-icon{ | |
| width:44px;height:44px;margin:0 auto var(--s-2); | |
| display:flex;align-items:center;justify-content:center; | |
| background:var(--surface);border:1px solid var(--border);border-radius:12px; | |
| color:var(--brand); | |
| } | |
| .dropzone-text{font-weight:600;font-size:15px} | |
| .dropzone-hint{color:var(--text-3);font-size:13px;margin-top:4px} | |
| .dropzone input{position:absolute;inset:0;opacity:0;cursor:pointer} | |
| .features{ | |
| display:grid;grid-template-columns:repeat(3,1fr); | |
| gap:var(--s-2);margin-top:var(--s-5);text-align:left; | |
| } | |
| .feature{ | |
| background:var(--surface-2); | |
| border:1px solid var(--border-soft); | |
| padding:var(--s-2) var(--s-3); | |
| border-radius:var(--r-sm); | |
| } | |
| .feature-title{font-weight:600;font-size:12px} | |
| .feature-desc{color:var(--text-2);font-size:12px;margin-top:2px;line-height:1.45} | |
| .powered-by{margin-top:var(--s-4);font-size:12px;color:var(--text-3)} | |
| .powered-by strong{color:var(--text-2);font-weight:600} | |
| /* =============== RESULTS VIEW =============== */ | |
| #results-view{display:none;min-height:100vh} | |
| /* ----- header ----- */ | |
| .topbar{ | |
| background:var(--surface); | |
| border-bottom:1px solid var(--border); | |
| padding:var(--s-2) var(--s-4); | |
| display:flex;align-items:center;gap:var(--s-3); | |
| position:sticky;top:0;z-index:50; | |
| min-height:64px; | |
| } | |
| .topbar .brand{gap:var(--s-2)} | |
| .topbar .brand-name{font-size:16px} | |
| .file-pill{ | |
| display:inline-flex;align-items:center;gap:var(--s-1); | |
| padding:6px var(--s-2); | |
| background:var(--surface-2); | |
| border:1px solid var(--border-soft); | |
| border-radius:999px; | |
| font-size:13px;color:var(--text-2);font-weight:500; | |
| max-width:380px; | |
| } | |
| .file-pill svg{flex-shrink:0;color:var(--text-3)} | |
| .file-pill-name{overflow:hidden;text-overflow:ellipsis;white-space:nowrap} | |
| .topbar-summary{font-size:13px;color:var(--text-3);margin-left:var(--s-1)} | |
| .topbar-spacer{flex:1} | |
| .topbar-actions{display:flex;gap:var(--s-1)} | |
| .btn{ | |
| display:inline-flex;align-items:center;gap:6px; | |
| padding:8px 14px;border-radius:var(--r-sm); | |
| font-weight:500;font-size:13px; | |
| border:1px solid transparent;cursor:pointer; | |
| transition:all .15s; | |
| background:transparent;color:var(--text); | |
| } | |
| .btn-ghost{background:var(--surface);border-color:var(--border)} | |
| .btn-ghost:hover{background:var(--surface-2);border-color:var(--text-3)} | |
| .btn-primary{ | |
| background:linear-gradient(135deg,var(--brand),var(--brand-2)); | |
| color:#fff;font-weight:600; | |
| box-shadow:0 2px 8px rgba(124,58,237,.22); | |
| } | |
| .btn-primary:hover{filter:brightness(1.06);box-shadow:0 4px 14px rgba(124,58,237,.28)} | |
| .btn svg{width:14px;height:14px} | |
| /* ----- summary cards ----- */ | |
| .metrics{ | |
| display:grid; | |
| grid-template-columns:repeat(4,minmax(0,1fr)); | |
| gap:var(--s-2); | |
| padding:var(--s-3) var(--s-4); | |
| background:var(--bg); | |
| } | |
| .metric{ | |
| background:var(--surface); | |
| border:1px solid var(--border); | |
| border-radius:var(--r-md); | |
| padding:var(--s-3); | |
| display:flex;flex-direction:column;gap:6px; | |
| position:relative;overflow:hidden; | |
| } | |
| .metric-label{ | |
| font-size:11px;font-weight:600; | |
| color:var(--text-3); | |
| letter-spacing:.06em;text-transform:uppercase; | |
| } | |
| .metric-value{ | |
| font-size:28px;font-weight:700;letter-spacing:-.02em; | |
| color:var(--text);font-variant-numeric:tabular-nums; | |
| } | |
| .metric-hint{font-size:12px;color:var(--text-3)} | |
| .metric-risk{display:inline-flex;align-items:center;gap:6px} | |
| .metric-risk .dot{width:10px;height:10px;border-radius:50%} | |
| .metric-risk.high .dot{background:var(--risk-high);box-shadow:0 0 0 4px rgba(185,28,28,.12)} | |
| .metric-risk.medium .dot{background:var(--risk-med);box-shadow:0 0 0 4px rgba(180,83,9,.12)} | |
| .metric-risk.low .dot{background:var(--risk-low);box-shadow:0 0 0 4px rgba(21,128,61,.12)} | |
| .metric-risk.high .lvl{color:var(--risk-high)} | |
| .metric-risk.medium .lvl{color:var(--risk-med)} | |
| .metric-risk.low .lvl{color:var(--risk-low)} | |
| /* ----- legend + distribution ----- */ | |
| .legend{ | |
| background:var(--surface); | |
| border:1px solid var(--border); | |
| border-radius:var(--r-md); | |
| margin:0 var(--s-4) var(--s-3); | |
| padding:var(--s-3); | |
| } | |
| .legend-header{ | |
| display:flex;align-items:center;justify-content:space-between; | |
| margin-bottom:var(--s-2); | |
| } | |
| .section-label{ | |
| font-size:11px;font-weight:600; | |
| color:var(--text-3); | |
| letter-spacing:.08em;text-transform:uppercase; | |
| } | |
| .dist-bar{ | |
| display:flex;height:6px;border-radius:999px;overflow:hidden; | |
| background:var(--border-soft);margin-bottom:var(--s-2); | |
| } | |
| .dist-seg{height:100%;transition:opacity .15s} | |
| .dist-seg:hover{opacity:.85} | |
| .chips{display:flex;flex-wrap:wrap;gap:6px} | |
| .chip{ | |
| display:inline-flex;align-items:center;gap:6px; | |
| padding:4px 10px; | |
| border-radius:999px; | |
| font-size:12px;font-weight:500; | |
| cursor:pointer;user-select:none; | |
| border:1px solid transparent; | |
| transition:all .15s; | |
| } | |
| .chip .chip-dot{width:7px;height:7px;border-radius:50%;flex-shrink:0} | |
| .chip .chip-count{ | |
| font-variant-numeric:tabular-nums; | |
| font-weight:600;opacity:.7;margin-left:2px; | |
| } | |
| .chip.inactive{opacity:.42;filter:grayscale(.3)} | |
| /* ----- layout ----- */ | |
| .layout{ | |
| display:grid; | |
| grid-template-columns:1fr 320px; | |
| gap:var(--s-3); | |
| padding:0 var(--s-4) var(--s-4); | |
| min-height:calc(100vh - 260px); | |
| } | |
| /* ----- document viewer ----- */ | |
| .doc-shell{ | |
| background:var(--surface); | |
| border:1px solid var(--border); | |
| border-radius:var(--r-lg); | |
| overflow:hidden; | |
| display:flex;flex-direction:column; | |
| min-height:600px; | |
| } | |
| .doc-toolbar{ | |
| display:flex;align-items:center;gap:var(--s-2); | |
| padding:var(--s-2) var(--s-3); | |
| border-bottom:1px solid var(--border-soft); | |
| background:var(--surface); | |
| } | |
| .seg{ | |
| display:inline-flex; | |
| background:var(--surface-2); | |
| border:1px solid var(--border-soft); | |
| border-radius:8px; | |
| padding:2px;gap:0; | |
| } | |
| .seg button{ | |
| padding:5px 12px; | |
| font-size:12px;font-weight:500; | |
| color:var(--text-2); | |
| background:transparent;border:none;border-radius:6px; | |
| cursor:pointer;transition:all .15s; | |
| } | |
| .seg button:hover{color:var(--text)} | |
| .seg button.active{ | |
| background:var(--surface); | |
| color:var(--text);font-weight:600; | |
| box-shadow:var(--shadow-xs); | |
| } | |
| .toolbar-divider{width:1px;height:20px;background:var(--border-soft)} | |
| .toolbar-spacer{flex:1} | |
| .icon-btn{ | |
| display:inline-flex;align-items:center;gap:6px; | |
| padding:5px 10px; | |
| font-size:12px;color:var(--text-2);font-weight:500; | |
| background:transparent;border:1px solid transparent;border-radius:6px; | |
| cursor:pointer;transition:all .15s; | |
| } | |
| .icon-btn:hover{background:var(--surface-2);color:var(--text)} | |
| .icon-btn svg{width:14px;height:14px} | |
| .icon-btn.toggle-on{background:var(--brand-soft);color:var(--brand);border-color:var(--brand-ring)} | |
| .doc-scroll{ | |
| flex:1;overflow-y:auto; | |
| background:var(--surface-warm); | |
| padding:var(--s-5) var(--s-4); | |
| } | |
| .doc-page{ | |
| background:var(--surface); | |
| border:1px solid var(--border-soft); | |
| border-radius:var(--r-xl); | |
| padding:var(--s-5) var(--s-6); | |
| max-width:820px;margin:0 auto; | |
| font-size:17px;line-height:1.75; | |
| color:#1e293b; | |
| white-space:pre-wrap;word-wrap:break-word; | |
| box-shadow:var(--shadow-sm); | |
| font-feature-settings:"liga","calt","tnum"; | |
| } | |
| .doc-page.focus-mode{color:rgba(30,41,59,.32)} | |
| .doc-page.focus-mode .pii{color:#1e293b} | |
| /* ----- PII spans (softer treatment) ----- */ | |
| .pii{ | |
| position:relative; | |
| padding:1px 4px; | |
| border-radius:4px; | |
| cursor:pointer; | |
| transition:all .15s ease; | |
| background:var(--pii-tint,rgba(0,0,0,.04)); | |
| color:var(--pii-text,inherit); | |
| box-shadow:inset 0 -1.5px 0 var(--pii-color,transparent); | |
| } | |
| .pii:hover{ | |
| background:var(--pii-color,#888); | |
| color:#fff; | |
| } | |
| .pii.dimmed{ | |
| opacity:.22; | |
| background:transparent; | |
| box-shadow:none; | |
| color:inherit; | |
| } | |
| .pii.linked{ | |
| box-shadow:0 0 0 2px var(--pii-color,#888), inset 0 -1.5px 0 var(--pii-color,transparent); | |
| background:var(--pii-color,#888);color:#fff; | |
| } | |
| .mask-token{ | |
| display:inline-block; | |
| padding:1px 8px; | |
| border-radius:4px; | |
| font-family:'JetBrains Mono',ui-monospace,monospace; | |
| font-size:13px;font-weight:600; | |
| background:var(--pii-tint,rgba(0,0,0,.06)); | |
| color:var(--pii-text,inherit); | |
| border:1px dashed var(--pii-color,#888); | |
| letter-spacing:.02em; | |
| } | |
| /* ----- sidebar (inspection rail) ----- */ | |
| .rail{ | |
| background:var(--surface); | |
| border:1px solid var(--border); | |
| border-radius:var(--r-lg); | |
| display:flex;flex-direction:column; | |
| overflow:hidden; | |
| max-height:calc(100vh - 260px); | |
| } | |
| .rail-section{ | |
| padding:var(--s-3); | |
| border-bottom:1px solid var(--border-soft); | |
| } | |
| .rail-section:last-child{border-bottom:none} | |
| .rail-section-header{ | |
| display:flex;align-items:center;justify-content:space-between; | |
| margin-bottom:var(--s-2); | |
| } | |
| .rail-section-header .section-label{margin:0} | |
| .rail-count{ | |
| font-size:11px;color:var(--text-3); | |
| font-variant-numeric:tabular-nums;font-weight:600; | |
| } | |
| .findings{display:flex;flex-direction:column;gap:2px} | |
| .finding{ | |
| display:flex;align-items:center;gap:var(--s-2); | |
| padding:8px 10px; | |
| border-radius:8px; | |
| cursor:pointer;user-select:none; | |
| transition:all .15s; | |
| border:1px solid transparent; | |
| } | |
| .finding:hover{background:var(--surface-2)} | |
| .finding.linked{background:var(--pii-tint);border-color:var(--pii-color)} | |
| .finding.inactive{opacity:.42} | |
| .finding-dot{width:8px;height:8px;border-radius:50%;flex-shrink:0;background:var(--pii-color,#888)} | |
| .finding-label{flex:1;font-size:13.5px;font-weight:500;color:var(--text)} | |
| .finding-count{ | |
| font-size:12px;font-weight:600;color:var(--text-2); | |
| background:var(--surface-2); | |
| padding:2px 8px;border-radius:999px; | |
| font-variant-numeric:tabular-nums; | |
| border:1px solid var(--border-soft); | |
| } | |
| .finding-toggle{ | |
| position:relative; | |
| width:26px;height:14px;border-radius:999px; | |
| background:var(--border); | |
| transition:background .18s;flex-shrink:0; | |
| } | |
| .finding.active .finding-toggle{background:var(--pii-color,var(--brand))} | |
| .finding-toggle::after{ | |
| content:'';position:absolute; | |
| top:1.5px;left:1.5px; | |
| width:11px;height:11px;border-radius:50%; | |
| background:#fff; | |
| transition:transform .18s; | |
| box-shadow:0 1px 2px rgba(0,0,0,.15); | |
| } | |
| .finding.active .finding-toggle::after{transform:translateX(12px)} | |
| .rail-actions{display:flex;flex-direction:column;gap:6px} | |
| .rail-actions .btn{justify-content:flex-start;width:100%} | |
| .rail-actions .btn-ghost{font-size:13px;padding:8px 12px} | |
| /* speakers */ | |
| .speakers{display:flex;flex-direction:column;gap:2px} | |
| .speaker{ | |
| display:flex;align-items:center;gap:var(--s-2); | |
| padding:6px 10px;border-radius:6px;cursor:pointer; | |
| font-size:13px;color:var(--text); | |
| transition:background .15s; | |
| } | |
| .speaker:hover{background:var(--surface-2)} | |
| .speaker-name{flex:1;font-weight:500} | |
| .speaker-count{ | |
| font-size:11px;color:var(--text-3);font-weight:600; | |
| font-variant-numeric:tabular-nums; | |
| } | |
| /* empty state */ | |
| .empty-state{ | |
| color:var(--text-3);font-size:13px;text-align:center; | |
| padding:var(--s-3); | |
| } | |
| /* tooltip */ | |
| .tooltip{ | |
| position:fixed; | |
| background:rgba(15,23,42,.95); | |
| color:#fff; | |
| padding:6px 10px;border-radius:6px; | |
| font-size:12px;font-weight:500; | |
| pointer-events:none;z-index:999; | |
| white-space:nowrap;max-width:320px;overflow:hidden;text-overflow:ellipsis; | |
| box-shadow:var(--shadow-md); | |
| backdrop-filter:blur(8px); | |
| } | |
| .tooltip .tt-label{ | |
| font-size:10px;text-transform:uppercase;letter-spacing:.06em; | |
| opacity:.7;margin-right:6px; | |
| } | |
| /* loading */ | |
| #loading{ | |
| position:fixed;inset:0; | |
| background:rgba(247,247,249,.86); | |
| backdrop-filter:blur(10px); | |
| display:none;flex-direction:column;align-items:center;justify-content:center; | |
| z-index:9999;gap:var(--s-2); | |
| } | |
| .spinner{ | |
| width:40px;height:40px; | |
| border:3px solid var(--border); | |
| border-top-color:var(--brand); | |
| border-radius:50%; | |
| animation:spin .7s linear infinite; | |
| } | |
| @keyframes spin{to{transform:rotate(360deg)}} | |
| #loading .load-title{font-size:14px;font-weight:600;color:var(--text);margin-top:var(--s-2)} | |
| #loading .load-sub{font-size:12px;color:var(--text-3)} | |
| .error-banner{ | |
| background:#fef2f2;border:1px solid #fecaca; | |
| color:#991b1b; | |
| padding:var(--s-2) var(--s-3); | |
| border-radius:var(--r-sm); | |
| margin:var(--s-3) var(--s-4); | |
| font-size:13px;display:none; | |
| } | |
| @media(max-width:960px){ | |
| .metrics{grid-template-columns:repeat(2,1fr)} | |
| .layout{grid-template-columns:1fr} | |
| .rail{max-height:none} | |
| .features{grid-template-columns:1fr} | |
| } | |
| </style> | |
| </head> | |
| <body> | |
| <!-- ============ UPLOAD VIEW ============ --> | |
| <div id="upload-view"> | |
| <div class="upload-card"> | |
| <div class="brand"> | |
| <div class="brand-logo">PR</div> | |
| <span class="brand-name">PII Reveal</span> | |
| </div> | |
| <div class="upload-hero">Document privacy inspection</div> | |
| <div class="upload-sub">Upload a document to detect and review sensitive content.</div> | |
| <div class="dropzone" id="dropzone"> | |
| <div class="dropzone-icon"> | |
| <svg width="22" height="22" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"> | |
| <path d="M12 3v12"/><path d="m7 8 5-5 5 5"/><path d="M21 15v4a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2v-4"/> | |
| </svg> | |
| </div> | |
| <div class="dropzone-text">Drop your document, or click to browse</div> | |
| <div class="dropzone-hint">PDF, DOC, DOCX · up to 128k tokens</div> | |
| <input type="file" id="file-input" accept=".pdf,.doc,.docx"> | |
| </div> | |
| <div class="features"> | |
| <div class="feature"><div class="feature-title">8 entity types</div><div class="feature-desc">Names, addresses, emails, phones, URLs, dates, accounts, secrets</div></div> | |
| <div class="feature"><div class="feature-title">128k context</div><div class="feature-desc">Full documents analyzed in a single pass</div></div> | |
| <div class="feature"><div class="feature-title">Context-aware</div><div class="feature-desc">Distinguishes “May” as a name vs. a month</div></div> | |
| </div> | |
| <div class="powered-by">Powered by <strong>OpenAI Privacy Filter</strong> · Apache 2.0</div> | |
| </div> | |
| </div> | |
| <!-- ============ RESULTS VIEW ============ --> | |
| <div id="results-view"> | |
| <!-- header --> | |
| <header class="topbar"> | |
| <div class="brand"> | |
| <div class="brand-logo">PR</div> | |
| <span class="brand-name">PII Reveal</span> | |
| </div> | |
| <div class="file-pill" id="file-pill"> | |
| <svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"> | |
| <path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"/><polyline points="14 2 14 8 20 8"/> | |
| </svg> | |
| <span class="file-pill-name" id="file-name"></span> | |
| </div> | |
| <span class="topbar-summary" id="topbar-summary"></span> | |
| <div class="topbar-spacer"></div> | |
| <div class="topbar-actions"> | |
| <button class="btn btn-ghost" id="btn-export-json"> | |
| <svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M21 15v4a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2v-4"/><polyline points="7 10 12 15 17 10"/><line x1="12" x2="12" y1="15" y2="3"/></svg> | |
| Export JSON | |
| </button> | |
| <button class="btn btn-ghost" id="btn-copy-masked"> | |
| <svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect width="14" height="14" x="8" y="8" rx="2"/><path d="M4 16c-1.1 0-2-.9-2-2V4c0-1.1.9-2 2-2h10c1.1 0 2 .9 2 2"/></svg> | |
| Copy masked | |
| </button> | |
| <button class="btn btn-primary" id="btn-new">New file</button> | |
| </div> | |
| </header> | |
| <div class="error-banner" id="error-banner"></div> | |
| <!-- KPI cards --> | |
| <section class="metrics"> | |
| <div class="metric"> | |
| <div class="metric-label">Sensitive content</div> | |
| <div class="metric-value" id="m-pct">0%</div> | |
| <div class="metric-hint" id="m-pct-hint">of document characters</div> | |
| </div> | |
| <div class="metric"> | |
| <div class="metric-label">Detected entities</div> | |
| <div class="metric-value" id="m-spans">0</div> | |
| <div class="metric-hint" id="m-spans-hint">spans flagged</div> | |
| </div> | |
| <div class="metric"> | |
| <div class="metric-label">Entity types</div> | |
| <div class="metric-value" id="m-cats">0</div> | |
| <div class="metric-hint" id="m-cats-hint">of 8 categories</div> | |
| </div> | |
| <div class="metric"> | |
| <div class="metric-label">Risk level</div> | |
| <div class="metric-value metric-risk" id="m-risk"> | |
| <span class="dot"></span><span class="lvl">—</span> | |
| </div> | |
| <div class="metric-hint" id="m-risk-hint">based on density & type</div> | |
| </div> | |
| </section> | |
| <!-- legend + distribution --> | |
| <section class="legend"> | |
| <div class="legend-header"> | |
| <span class="section-label">Detected categories</span> | |
| <span class="section-label" id="legend-total" style="color:var(--text-3)"></span> | |
| </div> | |
| <div class="dist-bar" id="dist-bar"></div> | |
| <div class="chips" id="chips"></div> | |
| </section> | |
| <!-- main layout --> | |
| <div class="layout"> | |
| <!-- document viewer --> | |
| <main class="doc-shell"> | |
| <div class="doc-toolbar"> | |
| <div class="seg" id="view-seg"> | |
| <button data-view="original" class="active">Original</button> | |
| <button data-view="masked">Masked</button> | |
| </div> | |
| <div class="toolbar-divider"></div> | |
| <button class="icon-btn" id="btn-focus" title="Dim everything except entities"> | |
| <svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><circle cx="12" cy="12" r="3"/><path d="M2 12s3-7 10-7 10 7 10 7-3 7-10 7-10-7-10-7Z"/></svg> | |
| Focus mode | |
| </button> | |
| <div class="toolbar-spacer"></div> | |
| <button class="icon-btn" id="btn-prev" title="Previous entity"> | |
| <svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="m15 18-6-6 6-6"/></svg> | |
| </button> | |
| <span id="nav-pos" style="font-size:12px;color:var(--text-3);font-variant-numeric:tabular-nums;min-width:52px;text-align:center">0 / 0</span> | |
| <button class="icon-btn" id="btn-next" title="Next entity"> | |
| <svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="m9 18 6-6-6-6"/></svg> | |
| </button> | |
| </div> | |
| <div class="doc-scroll" id="doc-scroll"> | |
| <div class="doc-page" id="doc-page"></div> | |
| </div> | |
| </main> | |
| <!-- inspection rail --> | |
| <aside class="rail"> | |
| <div class="rail-section"> | |
| <div class="rail-section-header"> | |
| <span class="section-label">Findings</span> | |
| <span class="rail-count" id="findings-count"></span> | |
| </div> | |
| <div class="findings" id="findings"></div> | |
| </div> | |
| <div class="rail-section" id="speakers-section" style="display:none"> | |
| <div class="rail-section-header"> | |
| <span class="section-label">Speakers</span> | |
| <span class="rail-count" id="speakers-count"></span> | |
| </div> | |
| <div class="speakers" id="speakers"></div> | |
| </div> | |
| <div class="rail-section"> | |
| <div class="rail-section-header"> | |
| <span class="section-label">Actions</span> | |
| </div> | |
| <div class="rail-actions"> | |
| <button class="btn btn-ghost" id="act-select-all">Select all categories</button> | |
| <button class="btn btn-ghost" id="act-clear-all">Clear selection</button> | |
| <button class="btn btn-ghost" id="act-copy-masked-2">Copy masked text</button> | |
| <button class="btn btn-ghost" id="act-export-json">Export findings (JSON)</button> | |
| </div> | |
| </div> | |
| </aside> | |
| </div> | |
| </div> | |
| <!-- loading --> | |
| <div id="loading"> | |
| <div class="spinner"></div> | |
| <div class="load-title">Analyzing document</div> | |
| <div class="load-sub">OpenAI Privacy Filter · 128k context</div> | |
| </div> | |
| <div class="tooltip" id="tooltip" style="display:none"></div> | |
| <script> | |
| /* ===== state ===== */ | |
| const S = { | |
| text:'', spans:[], stats:{}, speakers:{}, catMeta:{}, filename:'', | |
| activeCats:new Set(), activeSpeakers:new Set(), | |
| view:'original', // 'original' | 'masked' | |
| focus:false, | |
| sortedSpans:[], visibleIdx:[], navPos:-1, | |
| }; | |
| /* ===== labels (fallback when backend meta missing) ===== */ | |
| const LBL = {private_person:'Person',private_address:'Address',private_email:'Email',private_phone:'Phone',private_url:'URL',private_date:'Date',account_number:'Account',secret:'Secret'}; | |
| const COL = {private_person:'#dc2626',private_address:'#0891b2',private_email:'#2563eb',private_phone:'#16a34a',private_url:'#ca8a04',private_date:'#9333ea',account_number:'#ea580c',secret:'#b91c1c'}; | |
| const TINT = {private_person:'rgba(220,38,38,.08)',private_address:'rgba(8,145,178,.08)',private_email:'rgba(37,99,235,.08)',private_phone:'rgba(22,163,74,.08)',private_url:'rgba(202,138,4,.10)',private_date:'rgba(147,51,234,.08)',account_number:'rgba(234,88,12,.08)',secret:'rgba(185,28,28,.10)'}; | |
| const TEXT = {private_person:'#991b1b',private_address:'#155e75',private_email:'#1e40af',private_phone:'#14532d',private_url:'#713f12',private_date:'#6b21a8',account_number:'#7c2d12',secret:'#7f1d1d'}; | |
| const ORDER = ['private_person','private_email','private_phone','private_address','private_date','private_url','account_number','secret']; | |
| const metaFor = (c) => { | |
| const m = S.catMeta[c] || {}; | |
| return { | |
| color: m.color || COL[c] || '#64748b', | |
| tint: m.tint || TINT[c] || 'rgba(100,116,139,.08)', | |
| text: m.text || TEXT[c] || '#334155', | |
| label: m.label || LBL[c] || c, | |
| }; | |
| }; | |
| /* ===== upload flow ===== */ | |
| const dz = document.getElementById('dropzone'); | |
| const fi = document.getElementById('file-input'); | |
| ['dragenter','dragover'].forEach(e => dz.addEventListener(e, ev => { ev.preventDefault(); dz.classList.add('dragover'); })); | |
| ['dragleave','drop'].forEach(e => dz.addEventListener(e, ev => { ev.preventDefault(); dz.classList.remove('dragover'); })); | |
| dz.addEventListener('drop', ev => { if (ev.dataTransfer.files[0]) uploadFile(ev.dataTransfer.files[0]); }); | |
| fi.addEventListener('change', ev => { if (ev.target.files[0]) uploadFile(ev.target.files[0]); }); | |
| async function uploadFile(file) { | |
| const ext = file.name.split('.').pop().toLowerCase(); | |
| if (!['pdf','doc','docx'].includes(ext)) { showError('Unsupported file type.'); return; } | |
| document.getElementById('loading').style.display = 'flex'; | |
| document.getElementById('upload-view').style.display = 'none'; | |
| const form = new FormData(); form.append('file', file); | |
| try { | |
| const r = await fetch('/api/analyze', { method:'POST', body: form }); | |
| const d = await r.json(); | |
| if (d.error) { showError(d.error); return; } | |
| S.text = d.text; S.spans = d.spans; S.stats = d.stats; | |
| S.speakers = d.speakers || {}; S.catMeta = d.categories_meta || {}; | |
| S.filename = d.filename; | |
| S.activeCats = new Set(Object.keys(d.stats.categories)); | |
| S.activeSpeakers = new Set(Object.keys(S.speakers)); | |
| S.sortedSpans = [...S.spans].sort((a,b) => a.start - b.start); | |
| S.navPos = -1; | |
| renderResults(); | |
| } catch (e) { | |
| showError('Analysis failed: ' + e.message); | |
| } finally { | |
| document.getElementById('loading').style.display = 'none'; | |
| } | |
| } | |
| function showError(m) { | |
| document.getElementById('loading').style.display = 'none'; | |
| document.getElementById('results-view').style.display = 'block'; | |
| const b = document.getElementById('error-banner'); | |
| b.textContent = m; b.style.display = 'block'; | |
| } | |
| function resetView() { | |
| document.getElementById('results-view').style.display = 'none'; | |
| document.getElementById('upload-view').style.display = 'flex'; | |
| document.getElementById('error-banner').style.display = 'none'; | |
| fi.value = ''; | |
| } | |
| document.getElementById('btn-new').addEventListener('click', resetView); | |
| /* ===== render ===== */ | |
| function renderResults() { | |
| document.getElementById('results-view').style.display = 'block'; | |
| document.getElementById('error-banner').style.display = 'none'; | |
| document.getElementById('file-name').textContent = S.filename; | |
| document.getElementById('topbar-summary').textContent = | |
| `${S.stats.total_spans} entities across ${S.stats.num_categories} categories`; | |
| renderMetrics(); | |
| renderLegend(); | |
| renderFindings(); | |
| renderSpeakers(); | |
| renderDoc(); | |
| updateNavPos(); | |
| } | |
| function renderMetrics() { | |
| const s = S.stats; | |
| document.getElementById('m-pct').textContent = (s.pii_percentage ?? 0) + '%'; | |
| document.getElementById('m-pct-hint').textContent = `${s.pii_chars.toLocaleString()} of ${s.total_chars.toLocaleString()} chars`; | |
| document.getElementById('m-spans').textContent = s.total_spans; | |
| document.getElementById('m-spans-hint').textContent = 'spans flagged'; | |
| document.getElementById('m-cats').textContent = s.num_categories; | |
| document.getElementById('m-cats-hint').textContent = 'of 8 possible'; | |
| const risk = (s.risk_level || 'Low').toLowerCase(); | |
| const rEl = document.getElementById('m-risk'); | |
| rEl.className = 'metric-value metric-risk ' + risk; | |
| rEl.querySelector('.lvl').textContent = s.risk_level || 'Low'; | |
| } | |
| function renderLegend() { | |
| const s = S.stats, total = s.total_chars || 1; | |
| // distribution bar β only fills the fraction covered by PII | |
| const bar = document.getElementById('dist-bar'); | |
| bar.innerHTML = ''; | |
| const ordered = ORDER.filter(c => s.categories[c]); | |
| for (const c of ordered) { | |
| const info = s.categories[c], m = metaFor(c); | |
| const seg = document.createElement('div'); | |
| seg.className = 'dist-seg'; | |
| seg.style.width = (info.chars / total * 100) + '%'; | |
| seg.style.background = m.color; | |
| seg.dataset.cat = c; | |
| seg.title = `${m.label} Β· ${info.count} spans Β· ${info.chars} chars`; | |
| seg.addEventListener('mouseenter', () => highlightCategory(c, true)); | |
| seg.addEventListener('mouseleave', () => highlightCategory(c, false)); | |
| bar.appendChild(seg); | |
| } | |
| document.getElementById('legend-total').textContent = `${s.total_spans} entities`; | |
| // chips | |
| const ch = document.getElementById('chips'); ch.innerHTML = ''; | |
| for (const c of ordered) { | |
| const info = s.categories[c], m = metaFor(c); | |
| const el = document.createElement('span'); | |
| el.className = 'chip'; | |
| const active = S.activeCats.has(c); | |
| if (!active) el.classList.add('inactive'); | |
| el.style.background = m.tint; | |
| el.style.color = m.text; | |
| el.style.borderColor = 'transparent'; | |
| el.innerHTML = `<span class="chip-dot" style="background:${m.color}"></span>${m.label}<span class="chip-count">${info.count}</span>`; | |
| el.addEventListener('click', () => toggleCategory(c)); | |
| el.addEventListener('mouseenter', () => highlightCategory(c, true)); | |
| el.addEventListener('mouseleave', () => highlightCategory(c, false)); | |
| ch.appendChild(el); | |
| } | |
| } | |
| function renderFindings() { | |
| const box = document.getElementById('findings'); | |
| box.innerHTML = ''; | |
| const cats = S.stats.categories; | |
| const ordered = ORDER.filter(c => cats[c]); | |
| document.getElementById('findings-count').textContent = `${ordered.length} types`; | |
| if (!ordered.length) { box.innerHTML = '<div class="empty-state">No entities detected.</div>'; return; } | |
| for (const c of ordered) { | |
| const m = metaFor(c), info = cats[c], active = S.activeCats.has(c); | |
| const el = document.createElement('div'); | |
| el.className = 'finding' + (active ? ' active' : ' inactive'); | |
| el.dataset.cat = c; | |
| el.style.setProperty('--pii-color', m.color); | |
| el.style.setProperty('--pii-tint', m.tint); | |
| el.innerHTML = ` | |
| <span class="finding-dot"></span> | |
| <span class="finding-label">${m.label}</span> | |
| <span class="finding-count">${info.count}</span> | |
| <span class="finding-toggle"></span>`; | |
| el.addEventListener('click', () => toggleCategory(c)); | |
| el.addEventListener('mouseenter', () => highlightCategory(c, true)); | |
| el.addEventListener('mouseleave', () => highlightCategory(c, false)); | |
| box.appendChild(el); | |
| } | |
| } | |
| function renderSpeakers() { | |
| const sec = document.getElementById('speakers-section'), box = document.getElementById('speakers'); | |
| const names = Object.keys(S.speakers); | |
| if (!names.length) { sec.style.display = 'none'; return; } | |
| sec.style.display = 'block'; | |
| document.getElementById('speakers-count').textContent = `${names.length}`; | |
| box.innerHTML = ''; | |
| for (const name of names) { | |
| const el = document.createElement('div'); | |
| el.className = 'speaker'; | |
| el.innerHTML = `<span class="speaker-name">${esc(name)}</span><span class="speaker-count">${S.speakers[name]}</span>`; | |
| box.appendChild(el); | |
| } | |
| } | |
| function esc(s) { const d = document.createElement('div'); d.textContent = s; return d.innerHTML; } | |
| function renderDoc() { | |
| const { text, sortedSpans, view, activeCats, focus } = S; | |
| const page = document.getElementById('doc-page'); | |
| page.classList.toggle('focus-mode', focus); | |
| let html = '', pos = 0; | |
| S.visibleIdx = []; | |
| for (let i = 0; i < sortedSpans.length; i++) { | |
| const sp = sortedSpans[i]; | |
| if (sp.start < pos) continue; | |
| if (sp.start > pos) html += esc(text.substring(pos, sp.start)); | |
| const m = metaFor(sp.label); | |
| const on = activeCats.has(sp.label); | |
| if (on) S.visibleIdx.push(i); | |
| const style = `--pii-color:${m.color};--pii-tint:${m.tint};--pii-text:${m.text}`; | |
| if (view === 'masked' && on) { | |
| html += `<span class="mask-token" style="${style}" data-idx="${i}" data-cat="${sp.label}">[${m.label.toUpperCase()}]</span>`; | |
| } else { | |
| html += `<span class="pii${on ? '' : ' dimmed'}" style="${style}" data-idx="${i}" data-cat="${sp.label}" data-text="${esc(sp.text)}">${esc(text.substring(sp.start, sp.end))}</span>`; | |
| } | |
| pos = sp.end; | |
| } | |
| if (pos < text.length) html += esc(text.substring(pos)); | |
| page.innerHTML = html; | |
| const tt = document.getElementById('tooltip'); | |
| page.querySelectorAll('.pii, .mask-token').forEach(el => { | |
| const cat = el.dataset.cat, m = metaFor(cat); | |
| el.addEventListener('mouseenter', ev => { | |
| const orig = S.sortedSpans[+el.dataset.idx]; | |
| tt.innerHTML = `<span class="tt-label">${m.label}</span>${esc(orig.text)}`; | |
| tt.style.display = 'block'; moveTT(ev); | |
| highlightCategory(cat, true); | |
| }); | |
| el.addEventListener('mousemove', moveTT); | |
| el.addEventListener('mouseleave', () => { | |
| tt.style.display = 'none'; | |
| highlightCategory(cat, false); | |
| }); | |
| }); | |
| if (S.navPos >= S.visibleIdx.length) S.navPos = -1; | |
| } | |
| function moveTT(ev) { | |
| const t = document.getElementById('tooltip'); | |
| const w = t.offsetWidth || 200; | |
| const left = Math.min(ev.clientX + 12, window.innerWidth - w - 12); | |
| t.style.left = left + 'px'; | |
| t.style.top = (ev.clientY - 34) + 'px'; | |
| } | |
| /* ===== interactions ===== */ | |
| function toggleCategory(c) { | |
| if (S.activeCats.has(c)) S.activeCats.delete(c); | |
| else S.activeCats.add(c); | |
| renderLegend(); renderFindings(); renderDoc(); updateNavPos(); | |
| } | |
| function highlightCategory(c, on) { | |
| // span side | |
| document.querySelectorAll('.pii, .mask-token').forEach(el => { | |
| if (el.dataset.cat === c) el.classList.toggle('linked', on); | |
| }); | |
| // sidebar side | |
| document.querySelectorAll('.finding').forEach(el => { | |
| if (el.dataset.cat === c) el.classList.toggle('linked', on); | |
| }); | |
| } | |
| /* selection bulk actions */ | |
| document.getElementById('act-select-all').addEventListener('click', () => { | |
| S.activeCats = new Set(Object.keys(S.stats.categories)); | |
| renderLegend(); renderFindings(); renderDoc(); updateNavPos(); | |
| }); | |
| document.getElementById('act-clear-all').addEventListener('click', () => { | |
| S.activeCats = new Set(); | |
| renderLegend(); renderFindings(); renderDoc(); updateNavPos(); | |
| }); | |
| /* view segmented control */ | |
| document.getElementById('view-seg').addEventListener('click', ev => { | |
| const btn = ev.target.closest('button[data-view]'); | |
| if (!btn) return; | |
| S.view = btn.dataset.view; | |
| document.querySelectorAll('#view-seg button').forEach(b => b.classList.toggle('active', b === btn)); | |
| renderDoc(); | |
| }); | |
| /* focus mode */ | |
| document.getElementById('btn-focus').addEventListener('click', ev => { | |
| S.focus = !S.focus; | |
| ev.currentTarget.classList.toggle('toggle-on', S.focus); | |
| renderDoc(); | |
| }); | |
| /* next/prev */ | |
| document.getElementById('btn-next').addEventListener('click', () => navigate(1)); | |
| document.getElementById('btn-prev').addEventListener('click', () => navigate(-1)); | |
| function navigate(dir) { | |
| if (!S.visibleIdx.length) return; | |
| S.navPos = (S.navPos + dir + S.visibleIdx.length) % S.visibleIdx.length; | |
| const i = S.visibleIdx[S.navPos]; | |
| const el = document.querySelector(`[data-idx="${i}"]`); | |
| if (el) { | |
| el.scrollIntoView({ behavior:'smooth', block:'center' }); | |
| el.classList.add('linked'); | |
| setTimeout(() => el.classList.remove('linked'), 1200); | |
| } | |
| updateNavPos(); | |
| } | |
| function updateNavPos() { | |
| const total = S.visibleIdx.length; | |
| const cur = S.navPos >= 0 ? (S.navPos + 1) : 0; | |
| document.getElementById('nav-pos').textContent = `${cur} / ${total}`; | |
| } | |
| /* export + copy */ | |
| function maskedText() { | |
| const parts = []; let pos = 0; | |
| for (const sp of S.sortedSpans) { | |
| if (sp.start < pos) continue; | |
| if (sp.start > pos) parts.push(S.text.substring(pos, sp.start)); | |
| const m = metaFor(sp.label); | |
| parts.push(S.activeCats.has(sp.label) ? `[${m.label.toUpperCase()}]` : S.text.substring(sp.start, sp.end)); | |
| pos = sp.end; | |
| } | |
| if (pos < S.text.length) parts.push(S.text.substring(pos)); | |
| return parts.join(''); | |
| } | |
| function download(name, content, type = 'application/json') { | |
| const blob = new Blob([content], { type }); | |
| const a = document.createElement('a'); | |
| a.href = URL.createObjectURL(blob); a.download = name; | |
| document.body.appendChild(a); a.click(); a.remove(); | |
| setTimeout(() => URL.revokeObjectURL(a.href), 1000); | |
| } | |
| function exportJSON() { | |
| download((S.filename || 'findings') + '.findings.json', | |
| JSON.stringify({ filename:S.filename, stats:S.stats, spans:S.spans, speakers:S.speakers }, null, 2)); | |
| } | |
| async function copyMasked() { | |
| try { | |
| await navigator.clipboard.writeText(maskedText()); | |
| flashButton('Copied'); | |
| } catch { flashButton('Copy failed'); } | |
| } | |
| let _flashTimer; | |
| function flashButton(msg) { | |
| const b = document.getElementById('btn-copy-masked'); | |
| const prev = b.innerHTML; | |
| b.innerHTML = msg; | |
| clearTimeout(_flashTimer); | |
| _flashTimer = setTimeout(() => { b.innerHTML = prev; }, 1200); | |
| } | |
| document.getElementById('btn-export-json').addEventListener('click', exportJSON); | |
| document.getElementById('act-export-json').addEventListener('click', exportJSON); | |
| document.getElementById('btn-copy-masked').addEventListener('click', copyMasked); | |
| document.getElementById('act-copy-masked-2').addEventListener('click', copyMasked); | |
| /* keyboard: n / p for next/prev, f for focus */ | |
| document.addEventListener('keydown', ev => { | |
| if (document.getElementById('results-view').style.display === 'none') return; | |
| if (ev.target.matches('input,textarea')) return; | |
| if (ev.key === 'n' || ev.key === 'ArrowDown') { ev.preventDefault(); navigate(1); } | |
| else if (ev.key === 'p' || ev.key === 'ArrowUp') { ev.preventDefault(); navigate(-1); } | |
| else if (ev.key === 'f') { document.getElementById('btn-focus').click(); } | |
| }); | |
| </script> | |
| </body> | |
| </html>""" | |
| # ββ launch βββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| if __name__ == "__main__": | |
| server.launch(server_name="0.0.0.0", server_port=7860) | |