validate_sft.py

"""
LUNA 100M — SFT Validation on Complex Examples
================================================
Selects ~100 complex examples from the SFT validation set,
runs the fine-tuned model on each, and produces a detailed report.

Metrics computed:
  - Per-sample cross-entropy loss (prompt-masked) & perplexity
  - Token-level accuracy on the output portion
  - BLEU-1/2 (word overlap with reference output)
  - Repetition ratio (degeneration detection)
  - Response length stats
  - Category breakdown (coding, explanation, analysis, creative, how-to, identity)
  - Overall pass/fail grading

Usage:
    python validate_sft.py
    python validate_sft.py --ckpt "Base/out/sft/model.pth" --val_json "Base/Datasets/sft_clean/val.json"
"""

import os, sys, json, math, time, argparse, re
from pathlib import Path
from collections import Counter, defaultdict
from datetime import datetime

import torch
import torch.nn as nn
import torch.nn.functional as F


# ─── Model (identical to sft_train.py / chat.py) ─────────────────────────────

class RotaryEmbedding(nn.Module):
    def __init__(self, dim, max_seq_len=1024):
        super().__init__()
        inv_freq = 1.0 / (10000 ** (torch.arange(0, dim, 2).float() / dim))
        self.register_buffer("inv_freq", inv_freq)
        t = torch.arange(max_seq_len).float()
        freqs = torch.einsum("i,j->ij", t, inv_freq)
        emb = torch.cat([freqs, freqs], dim=-1)
        self.register_buffer("cos_cached", emb.cos())
        self.register_buffer("sin_cached", emb.sin())

    def forward(self, seq_len):
        return self.cos_cached[:seq_len], self.sin_cached[:seq_len]


def rotate_half(x):
    x1, x2 = x.chunk(2, dim=-1)
    return torch.cat([-x2, x1], dim=-1)


def apply_rotary(x, cos, sin):
    c = cos.unsqueeze(0).unsqueeze(0)
    s = sin.unsqueeze(0).unsqueeze(0)
    return x * c + rotate_half(x) * s


class CausalSelfAttention(nn.Module):
    def __init__(self, n_embd, n_head, block_size, rotary_pct=0.25):
        super().__init__()
        self.n_head = n_head
        self.head_dim = n_embd // n_head
        self.rot_dim = int(self.head_dim * rotary_pct)
        self.c_attn = nn.Linear(n_embd, 3 * n_embd, bias=True)
        self.c_proj = nn.Linear(n_embd, n_embd, bias=True)
        self.rotary = RotaryEmbedding(self.rot_dim, block_size)

    def forward(self, x):
        B, T, C = x.size()
        qkv = self.c_attn(x).reshape(B, T, 3, self.n_head, self.head_dim).permute(2, 0, 3, 1, 4)
        q, k, v = qkv.unbind(0)
        cos, sin = self.rotary(T)
        q = torch.cat([apply_rotary(q[..., :self.rot_dim], cos, sin), q[..., self.rot_dim:]], dim=-1)
        k = torch.cat([apply_rotary(k[..., :self.rot_dim], cos, sin), k[..., self.rot_dim:]], dim=-1)
        y = F.scaled_dot_product_attention(q, k, v, is_causal=True)
        return self.c_proj(y.transpose(1, 2).contiguous().view(B, T, C))


class MLP(nn.Module):
    def __init__(self, n_embd):
        super().__init__()
        self.fc = nn.Linear(n_embd, 4 * n_embd, bias=True)
        self.gelu = nn.GELU()
        self.proj = nn.Linear(4 * n_embd, n_embd, bias=True)

    def forward(self, x):
        return self.proj(self.gelu(self.fc(x)))


class Block(nn.Module):
    def __init__(self, n_embd, n_head, block_size):
        super().__init__()
        self.ln1 = nn.LayerNorm(n_embd)
        self.attn = CausalSelfAttention(n_embd, n_head, block_size)
        self.ln2 = nn.LayerNorm(n_embd)
        self.mlp = MLP(n_embd)

    def forward(self, x):
        x = x + self.attn(self.ln1(x))
        x = x + self.mlp(self.ln2(x))
        return x


class LUNAModel(nn.Module):
    def __init__(self, vocab_size=50304, block_size=1024,
                 n_layer=10, n_embd=768, n_head=12):
        super().__init__()
        self.block_size = block_size
        self.wte = nn.Embedding(vocab_size, n_embd)
        self.blocks = nn.ModuleList(
            [Block(n_embd, n_head, block_size) for _ in range(n_layer)]
        )
        self.ln_f = nn.LayerNorm(n_embd)
        self.lm_head = nn.Linear(n_embd, vocab_size, bias=False)
        self.lm_head.weight = self.wte.weight

    def forward(self, idx):
        x = self.wte(idx)
        for block in self.blocks:
            x = block(x)
        return self.lm_head(self.ln_f(x))


# ─── Generation ───────────────────────────────────────────────────────────────

@torch.no_grad()
def generate(model, input_ids, max_new=150, temperature=0.7,
             top_p=0.9, top_k=40, repetition_penalty=1.0, device="cpu"):
    ids = input_ids.to(device)
    generated = []
    for _ in range(max_new):
        logits = model(ids[:, -model.block_size:])[:, -1, :]
        if repetition_penalty != 1.0:
            for tok_id in set(ids[0].tolist()):
                if logits[0, tok_id] > 0:
                    logits[0, tok_id] /= repetition_penalty
                else:
                    logits[0, tok_id] *= repetition_penalty
        if temperature < 1e-6:
            next_token = logits.argmax(dim=-1, keepdim=True)
        else:
            logits = logits / temperature
            probs = F.softmax(logits, dim=-1)
            if top_k > 0:
                kval = min(top_k, probs.size(-1))
                topk_vals, _ = torch.topk(probs, kval)
                probs[probs < topk_vals[:, [-1]]] = 0.0
                probs /= probs.sum()
            if top_p < 1.0:
                sorted_probs, sorted_idx = torch.sort(probs, descending=True)
                cumsum = torch.cumsum(sorted_probs, dim=-1)
                mask = cumsum - sorted_probs > top_p
                sorted_probs[mask] = 0.0
                sorted_probs /= sorted_probs.sum()
                next_token = sorted_idx[0, torch.multinomial(sorted_probs[0], 1)]
            else:
                next_token = torch.multinomial(probs[0], 1)
        ids = torch.cat([ids, next_token.view(1, 1)], dim=1)
        generated.append(next_token.item())
        if next_token.item() == 0:   # EOS token
            break
    return generated


# ─── Prompt formatting (matches sft_train.py) ────────────────────────────────

def format_prompt(instruction, inp=""):
    inst = instruction.strip()
    inp = inp.strip()
    if inst and inp:
        return f"### Instruction:\n{inst}\n\n### Input:\n{inp}\n\n### Response:\n"
    elif inst:
        return f"### Instruction:\n{inst}\n\n### Response:\n"
    else:
        return f"### Input:\n{inp}\n\n### Response:\n"


# ─── Complexity scoring & selection ───────────────────────────────────────────

COMPLEXITY_KEYWORDS = [
    "step", "first", "second", "then", "next", "finally",
    "because", "however", "therefore", "explain", "analyze",
    "compare", "evaluate", "describe", "discuss", "provide",
    "example", "detail", "elaborate", "summarize",
]

def complexity_score(entry):
    inst = entry.get("instruction", "")
    inp = entry.get("input", "")
    out = entry.get("output", "")
    total_text = (inst + " " + inp + " " + out).lower()
    total_len = len(inst) + len(inp) + len(out)
    has_input = 1 if len(inp) > 20 else 0
    kw_count = sum(1 for w in COMPLEXITY_KEYWORDS if w in total_text)
    return total_len * 0.3 + len(out) * 0.5 + has_input * 500 + kw_count * 200


def categorize(instruction):
    inst = instruction.lower()
    if any(w in inst for w in ["code", "python", "java", "swift", "function", "program", "algorithm", "script", "sql", "html", "css"]):
        return "coding"
    if any(w in inst for w in ["who are you", "your name", "who created", "asterizer", "luna", "are you an ai"]):
        return "identity"
    if any(w in inst for w in ["explain", "what is", "define", "describe", "meaning of"]):
        return "explanation"
    if any(w in inst for w in ["analyze", "compare", "evaluate", "assess", "critique"]):
        return "analysis"
    if any(w in inst for w in ["write", "create", "generate", "compose", "draft", "poem", "story", "essay"]):
        return "creative"
    if any(w in inst for w in ["how", "step", "guide", "method", "procedure", "tutorial"]):
        return "how-to"
    return "other"


def select_complex_examples(data, n=100):
    scored = [(complexity_score(entry), i) for i, entry in enumerate(data)]
    scored.sort(reverse=True)
    return [data[idx] for _, idx in scored[:n]]


# ─── Metrics ──────────────────────────────────────────────────────────────────

def compute_bleu(reference, hypothesis, max_n=2):
    """Simple BLEU-1 and BLEU-2 (word-level, no brevity penalty)."""
    ref_tokens = reference.lower().split()
    hyp_tokens = hypothesis.lower().split()
    if not hyp_tokens or not ref_tokens:
        return {f"bleu_{n}": 0.0 for n in range(1, max_n + 1)}
    scores = {}
    for n in range(1, max_n + 1):
        ref_ngrams = Counter()
        for i in range(len(ref_tokens) - n + 1):
            ref_ngrams[tuple(ref_tokens[i:i + n])] += 1
        hyp_ngrams = Counter()
        for i in range(len(hyp_tokens) - n + 1):
            hyp_ngrams[tuple(hyp_tokens[i:i + n])] += 1
        clipped = sum(min(hyp_ngrams[ng], ref_ngrams[ng]) for ng in hyp_ngrams)
        total = max(sum(hyp_ngrams.values()), 1)
        scores[f"bleu_{n}"] = clipped / total
    return scores


def repetition_ratio(text):
    """Fraction of repeated trigrams in the text (higher = more degenerate)."""
    words = text.lower().split()
    if len(words) < 4:
        return 0.0
    trigrams = [tuple(words[i:i + 3]) for i in range(len(words) - 2)]
    if not trigrams:
        return 0.0
    unique = len(set(trigrams))
    return 1.0 - (unique / len(trigrams))


@torch.no_grad()
def compute_loss_and_accuracy(model, tokenizer, entry, max_len, device):
    """Compute prompt-masked CE loss & token accuracy for one example."""
    prompt = format_prompt(entry.get("instruction", ""), entry.get("input", ""))
    response = entry.get("output", "").strip()

    prompt_ids = tokenizer.encode(prompt)
    response_ids = tokenizer.encode(response) + [tokenizer.eos_token_id or 0]
    total_ids = prompt_ids + response_ids

    if len(total_ids) > max_len:
        total_ids = total_ids[:max_len]
        total_ids[-1] = tokenizer.eos_token_id or 0
        prompt_len = min(len(prompt_ids), max_len)
    else:
        prompt_len = len(prompt_ids)

    input_tensor = torch.tensor([total_ids], dtype=torch.long, device=device)
    logits = model(input_tensor)  # (1, T, V)

    # Shift for next-token prediction
    shift_logits = logits[:, :-1, :].contiguous()
    shift_targets = input_tensor[:, 1:].contiguous()

    # Build mask: only on response portion
    mask = torch.zeros(shift_targets.shape, dtype=torch.float, device=device)
    resp_start = max(prompt_len - 1, 0)
    resp_end = len(total_ids) - 1
    mask[0, resp_start:resp_end] = 1.0

    if mask.sum() == 0:
        return float("inf"), 0.0, 0

    per_token_loss = F.cross_entropy(
        shift_logits.view(-1, shift_logits.size(-1)),
        shift_targets.view(-1),
        reduction="none"
    ).view(shift_targets.shape)

    masked_loss = (per_token_loss * mask).sum() / mask.sum()

    # Token accuracy on response portion
    preds = shift_logits.argmax(dim=-1)
    correct = ((preds == shift_targets).float() * mask).sum()
    total_resp = mask.sum()
    accuracy = (correct / total_resp).item() if total_resp > 0 else 0.0

    return masked_loss.item(), accuracy, int(total_resp.item())


# ─── Main validation ─────────────────────────────────────────────────────────

def main():
    parser = argparse.ArgumentParser(description="LUNA SFT — Complex Example Validation")
    parser.add_argument("--ckpt", default=r"D:\ASTERIZER 2026\LUNA\Base\out\sft\model.pth")
    parser.add_argument("--tok_dir", default="Base/checkpoints/EleutherAI/pythia-160m")
    parser.add_argument("--val_json", default="Base/Datasets/sft_clean/val.json")
    parser.add_argument("--n_examples", type=int, default=100)
    parser.add_argument("--max_len", type=int, default=1024)
    parser.add_argument("--max_new", type=int, default=150)
    parser.add_argument("--temperature", type=float, default=0.7)
    parser.add_argument("--top_k", type=int, default=40)
    parser.add_argument("--top_p", type=float, default=0.9)
    parser.add_argument("--rep_pen", type=float, default=1.0)
    parser.add_argument("--device", default="auto")
    parser.add_argument("--out_dir", default="Base/out/sft/validation_report_v2")
    args = parser.parse_args()

    device = "cuda" if args.device == "auto" and torch.cuda.is_available() else args.device
    if device == "auto":
        device = "cpu"

    out_dir = Path(args.out_dir)
    out_dir.mkdir(parents=True, exist_ok=True)

    sep = "=" * 72
    print(f"\n{sep}")
    print("  LUNA 100M — SFT VALIDATION (Complex Examples)")
    print(sep)
    print(f"  Checkpoint : {args.ckpt}")
    print(f"  Val data   : {args.val_json}")
    print(f"  N examples : {args.n_examples}")
    print(f"  Device     : {device}")
    print(f"  Max seq    : {args.max_len}")
    print(f"  Temperature: {args.temperature}")
    print(f"  Top-k      : {args.top_k}")
    print(f"  Top-p      : {args.top_p}")
    print(f"  Rep penalty: {args.rep_pen}")
    print(sep)

    # ── Load model ────────────────────────────────────────────────────────────
    print("\n[1/5] Loading model...")
    t0 = time.time()
    state_dict = torch.load(args.ckpt, map_location="cpu", weights_only=True)
    if isinstance(state_dict, dict) and "model" in state_dict:
        state_dict = state_dict["model"]
    model = LUNAModel()
    model.load_state_dict(state_dict, strict=True)
    model = model.to(device).eval()
    n_params = sum(p.numel() for p in model.parameters())
    print(f"      Model loaded: {n_params:,} params ({time.time()-t0:.1f}s)")

    # ── Load tokenizer ────────────────────────────────────────────────────────
    print("[2/5] Loading tokenizer...")
    from transformers import AutoTokenizer
    tokenizer = AutoTokenizer.from_pretrained(args.tok_dir)
    print(f"      Tokenizer: vocab_size={tokenizer.vocab_size}")

    # ── Select complex examples ───────────────────────────────────────────────
    print(f"[3/5] Selecting {args.n_examples} most complex examples from val set...")
    with open(args.val_json, "r", encoding="utf-8") as f:
        all_val = json.load(f)
    print(f"      Total val samples: {len(all_val)}")
    examples = select_complex_examples(all_val, args.n_examples)
    print(f"      Selected: {len(examples)} complex examples")

    # Category breakdown
    cat_counts = Counter(categorize(e["instruction"]) for e in examples)
    print(f"      Categories: {dict(cat_counts)}")

    # ── Run validation ────────────────────────────────────────────────────────
    print(f"\n[4/5] Running validation ({len(examples)} examples)...")
    print("-" * 72)

    results = []
    cat_metrics = defaultdict(lambda: {"losses": [], "perplexities": [],
                                        "accuracies": [], "bleu1": [],
                                        "bleu2": [], "rep_ratios": [],
                                        "gen_lens": []})

    for i, entry in enumerate(examples):
        inst = entry.get("instruction", "")
        inp = entry.get("input", "")
        ref_output = entry.get("output", "")
        category = categorize(inst)

        # 1) Compute loss & accuracy (teacher-forced)
        loss, tok_acc, n_resp_tokens = compute_loss_and_accuracy(
            model, tokenizer, entry, args.max_len, device
        )
        ppl = math.exp(min(loss, 20))  # cap to avoid overflow

        # 2) Generate response (autoregressive)
        prompt = format_prompt(inst, inp)
        prompt_ids = tokenizer.encode(prompt, return_tensors="pt")
        gen_tokens = generate(
            model, prompt_ids,
            max_new=args.max_new,
            temperature=args.temperature,
            top_k=args.top_k,
            top_p=args.top_p,
            repetition_penalty=args.rep_pen,
            device=device,
        )
        gen_text = tokenizer.decode(gen_tokens, skip_special_tokens=True).strip()
        # Clean trailing template markers
        if "### " in gen_text:
            gen_text = gen_text.split("### ")[0].strip()

        # 3) Compute text metrics
        bleu = compute_bleu(ref_output, gen_text)
        rep = repetition_ratio(gen_text)
        gen_words = len(gen_text.split())

        # 4) Quality flags
        is_empty = len(gen_text.strip()) < 5
        is_repetitive = rep > 0.5
        is_truncated = len(gen_tokens) >= args.max_new

        result = {
            "index": i,
            "category": category,
            "instruction": inst[:200],
            "input_preview": inp[:100] if inp else "",
            "reference_preview": ref_output[:200],
            "generated_preview": gen_text[:300],
            "loss": round(loss, 4),
            "perplexity": round(ppl, 2),
            "token_accuracy": round(tok_acc, 4),
            "bleu_1": round(bleu["bleu_1"], 4),
            "bleu_2": round(bleu["bleu_2"], 4),
            "repetition_ratio": round(rep, 4),
            "generated_words": gen_words,
            "resp_tokens": n_resp_tokens,
            "is_empty": is_empty,
            "is_repetitive": is_repetitive,
            "is_truncated": is_truncated,
        }
        results.append(result)

        # Accumulate per-category
        cat_metrics[category]["losses"].append(loss)
        cat_metrics[category]["perplexities"].append(ppl)
        cat_metrics[category]["accuracies"].append(tok_acc)
        cat_metrics[category]["bleu1"].append(bleu["bleu_1"])
        cat_metrics[category]["bleu2"].append(bleu["bleu_2"])
        cat_metrics[category]["rep_ratios"].append(rep)
        cat_metrics[category]["gen_lens"].append(gen_words)

        # Progress
        status = ""
        if is_empty:
            status = " [EMPTY]"
        elif is_repetitive:
            status = " [REPETITIVE]"
        elif is_truncated:
            status = " [TRUNCATED]"

        if (i + 1) % 5 == 0 or i == 0:
            print(f"  [{i+1:3d}/{len(examples)}] loss={loss:.3f} ppl={ppl:.1f} "
                  f"acc={tok_acc:.3f} B1={bleu['bleu_1']:.3f} "
                  f"rep={rep:.3f} words={gen_words}{status}")

    # ── Aggregate & Report ────────────────────────────────────────────────────
    print(f"\n[5/5] Generating report...")

    all_losses = [r["loss"] for r in results if r["loss"] < float("inf")]
    all_ppls = [r["perplexity"] for r in results if r["perplexity"] < 1e6]
    all_accs = [r["token_accuracy"] for r in results]
    all_b1 = [r["bleu_1"] for r in results]
    all_b2 = [r["bleu_2"] for r in results]
    all_reps = [r["repetition_ratio"] for r in results]
    all_lens = [r["generated_words"] for r in results]

    n_empty = sum(1 for r in results if r["is_empty"])
    n_repetitive = sum(1 for r in results if r["is_repetitive"])
    n_truncated = sum(1 for r in results if r["is_truncated"])

    avg = lambda xs: sum(xs) / len(xs) if xs else 0.0

    # ── Build report text ─────────────────────────────────────────────────────
    report_lines = []
    def P(s=""):
        report_lines.append(s)

    P(sep)
    P("  LUNA 100M — SFT VALIDATION REPORT")
    P(f"  Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
    P(sep)
    P()
    P(f"  Checkpoint    : {args.ckpt}")
    P(f"  Val source    : {args.val_json} ({len(all_val)} total samples)")
    P(f"  Examples tested: {len(examples)} (top complex by scoring)")
    P(f"  Device        : {device}")
    P(f"  Max seq len   : {args.max_len}")
    P(f"  Gen temp      : {args.temperature}")
    P(f"  Gen top_k     : {args.top_k}")
    P(f"  Gen top_p     : {args.top_p}")
    P(f"  Gen rep_pen   : {args.rep_pen}")
    P(f"  Gen max tokens: {args.max_new}")
    P()

    P("=" * 72)
    P("  OVERALL METRICS")
    P("=" * 72)
    P(f"  Avg Loss (CE)         : {avg(all_losses):.4f}")
    P(f"  Avg Perplexity        : {avg(all_ppls):.2f}")
    P(f"  Median Perplexity     : {sorted(all_ppls)[len(all_ppls)//2]:.2f}" if all_ppls else "  Median Perplexity     : N/A")
    P(f"  Avg Token Accuracy    : {avg(all_accs):.4f}  ({avg(all_accs)*100:.1f}%)")
    P(f"  Avg BLEU-1            : {avg(all_b1):.4f}")
    P(f"  Avg BLEU-2            : {avg(all_b2):.4f}")
    P(f"  Avg Repetition Ratio  : {avg(all_reps):.4f}")
    P(f"  Avg Gen Length (words): {avg(all_lens):.1f}")
    P()
    P(f"  Empty responses       : {n_empty}/{len(results)}")
    P(f"  Repetitive responses  : {n_repetitive}/{len(results)}")
    P(f"  Truncated responses   : {n_truncated}/{len(results)}")
    P()

    # Quality grade
    grade = "A"
    grade_notes = []
    if avg(all_ppls) > 50:
        grade = "C"
        grade_notes.append("high perplexity (>50)")
    elif avg(all_ppls) > 20:
        grade = "B"
        grade_notes.append("moderate perplexity (>20)")
    if n_empty > 10:
        grade = "D" if grade > "C" else grade
        grade_notes.append(f"{n_empty} empty responses")
    elif n_empty > 3:
        grade = max(grade, "C")
        grade_notes.append(f"{n_empty} empty responses")
    if n_repetitive > 15:
        grade = max(grade, "C")
        grade_notes.append(f"{n_repetitive} repetitive responses")
    if avg(all_b1) < 0.05:
        grade = max(grade, "C")
        grade_notes.append("very low BLEU-1")
    if avg(all_accs) > 0.4:
        grade_notes.append("strong token accuracy")
    if avg(all_accs) > 0.5:
        if grade == "B":
            grade = "A-"

    P(f"  OVERALL GRADE: {grade}")
    if grade_notes:
        P(f"  Notes: {'; '.join(grade_notes)}")
    P()

    P("=" * 72)
    P("  CATEGORY BREAKDOWN")
    P("=" * 72)
    P(f"  {'Category':<14} {'Count':>5} {'Avg Loss':>9} {'Avg PPL':>9} "
      f"{'Avg Acc':>8} {'BLEU-1':>7} {'BLEU-2':>7} {'Rep %':>6}")
    P("  " + "-" * 68)
    for cat in sorted(cat_metrics.keys()):
        m = cat_metrics[cat]
        cnt = len(m["losses"])
        P(f"  {cat:<14} {cnt:>5} {avg(m['losses']):>9.4f} "
          f"{avg(m['perplexities']):>9.2f} {avg(m['accuracies']):>8.4f} "
          f"{avg(m['bleu1']):>7.4f} {avg(m['bleu2']):>7.4f} "
          f"{avg(m['rep_ratios'])*100:>5.1f}%")
    P()

    # ── Top 5 Best / Worst ────────────────────────────────────────────────────
    P("=" * 72)
    P("  TOP 5 BEST (lowest perplexity)")
    P("=" * 72)
    by_ppl = sorted(results, key=lambda r: r["perplexity"])
    for r in by_ppl[:5]:
        P(f"  [{r['index']:3d}] PPL={r['perplexity']:>8.2f}  Acc={r['token_accuracy']:.3f}  "
          f"B1={r['bleu_1']:.3f}  [{r['category']}]")
        P(f"        Q: {r['instruction'][:80]}")
        P(f"        A: {r['generated_preview'][:100]}")
        P()

    P("=" * 72)
    P("  TOP 5 WORST (highest perplexity)")
    P("=" * 72)
    for r in by_ppl[-5:]:
        P(f"  [{r['index']:3d}] PPL={r['perplexity']:>8.2f}  Acc={r['token_accuracy']:.3f}  "
          f"B1={r['bleu_1']:.3f}  [{r['category']}]")
        P(f"        Q: {r['instruction'][:80]}")
        P(f"        A: {r['generated_preview'][:100]}")
        P()

    # ── Failure Analysis ──────────────────────────────────────────────────────
    failures = [r for r in results if r["is_empty"] or r["is_repetitive"]]
    if failures:
        P("=" * 72)
        P(f"  FAILURE ANALYSIS ({len(failures)} problematic responses)")
        P("=" * 72)
        for r in failures[:10]:
            flags = []
            if r["is_empty"]:
                flags.append("EMPTY")
            if r["is_repetitive"]:
                flags.append("REPETITIVE")
            P(f"  [{r['index']:3d}] {' | '.join(flags)}  [{r['category']}]")
            P(f"        Q: {r['instruction'][:80]}")
            P(f"        A: {r['generated_preview'][:120]}")
            P()

    # ── Perplexity distribution ───────────────────────────────────────────────
    P("=" * 72)
    P("  PERPLEXITY DISTRIBUTION")
    P("=" * 72)
    buckets = [(0, 5), (5, 10), (10, 20), (20, 50), (50, 100),
               (100, 500), (500, float("inf"))]
    for lo, hi in buckets:
        cnt = sum(1 for p in all_ppls if lo <= p < hi)
        bar = "#" * cnt
        label = f"{lo}-{hi}" if hi != float("inf") else f"{lo}+"
        P(f"  {label:>8}: {cnt:>3} {bar}")
    P()

    # ── Sample generations (10 diverse examples) ──────────────────────────────
    P("=" * 72)
    P("  SAMPLE GENERATIONS (10 diverse examples)")
    P("=" * 72)
    # Pick every 10th
    sample_indices = list(range(0, len(results), max(1, len(results) // 10)))[:10]
    for si in sample_indices:
        r = results[si]
        P(f"\n  --- Example {r['index']+1} [{r['category']}] ---")
        P(f"  Instruction: {r['instruction'][:150]}")
        if r["input_preview"]:
            P(f"  Input: {r['input_preview'][:100]}")
        P(f"  Reference: {r['reference_preview'][:200]}")
        P(f"  Generated: {r['generated_preview'][:300]}")
        P(f"  Loss={r['loss']:.4f}  PPL={r['perplexity']:.2f}  "
          f"Acc={r['token_accuracy']:.4f}  BLEU-1={r['bleu_1']:.4f}  "
          f"Rep={r['repetition_ratio']:.4f}")
    P()

    P(sep)
    P("  END OF REPORT")
    P(sep)

    report_text = "\n".join(report_lines)

    # Print to console
    print(report_text)

    # Save report
    report_path = out_dir / "SFT_VALIDATION_REPORT.txt"
    with open(report_path, "w", encoding="utf-8") as f:
        f.write(report_text)
    print(f"\n  Report saved: {report_path}")

    # Save detailed JSON results
    json_path = out_dir / "validation_results.json"
    summary = {
        "meta": {
            "checkpoint": args.ckpt,
            "val_source": args.val_json,
            "total_val_samples": len(all_val),
            "n_tested": len(examples),
            "device": device,
            "max_len": args.max_len,
            "temperature": args.temperature,
            "top_k": args.top_k,
            "top_p": args.top_p,
            "repetition_penalty": args.rep_pen,
            "timestamp": datetime.now().isoformat(),
        },
        "overall": {
            "avg_loss": round(avg(all_losses), 4),
            "avg_perplexity": round(avg(all_ppls), 2),
            "median_perplexity": round(sorted(all_ppls)[len(all_ppls)//2], 2) if all_ppls else None,
            "avg_token_accuracy": round(avg(all_accs), 4),
            "avg_bleu_1": round(avg(all_b1), 4),
            "avg_bleu_2": round(avg(all_b2), 4),
            "avg_repetition_ratio": round(avg(all_reps), 4),
            "avg_gen_length_words": round(avg(all_lens), 1),
            "n_empty": n_empty,
            "n_repetitive": n_repetitive,
            "n_truncated": n_truncated,
            "grade": grade,
        },
        "category_breakdown": {
            cat: {
                "count": len(m["losses"]),
                "avg_loss": round(avg(m["losses"]), 4),
                "avg_perplexity": round(avg(m["perplexities"]), 2),
                "avg_token_accuracy": round(avg(m["accuracies"]), 4),
                "avg_bleu_1": round(avg(m["bleu1"]), 4),
                "avg_bleu_2": round(avg(m["bleu2"]), 4),
            }
            for cat, m in cat_metrics.items()
        },
        "per_example": results,
    }
    with open(json_path, "w", encoding="utf-8") as f:
        json.dump(summary, f, indent=2, ensure_ascii=False)
    print(f"  JSON results: {json_path}")


if __name__ == "__main__":
    main()