import { useState, useRef, useEffect, useCallback } from "react"; import type { Affect, Candidate, ChatMessage, LatencyLog, SensingState, } from "../types"; import { pollEvals, sendPick, sendTurnaround, streamChat, streamRegenerate, } from "../lib/api"; import { EvalPanel } from "./EvalPanel"; import { useVoice } from "../hooks/useVoice"; import { isVoiceCapable } from "../lib/voiceEligibility"; import { resolveIntent } from "../lib/resolveIntent"; const STRATEGY_LABELS: Record = { broad: "broad — all memories", focused: "focused — top memory", serendipitous: "serendipitous — other memory", side_index: "like last time", present_good: "feeling good", present_fine: "doing okay", present_rough: "not great", pending: "", }; interface Props { userId: string | null; personaName: string; sensing: SensingState; affectOverride: Affect | null; onAirTextConsumed: () => void; onHeadSignalConsumed: () => void; messages: ChatMessage[]; setMessages: React.Dispatch>; onLatency: (latency: LatencyLog) => void; backendReady: boolean; } const TURNAROUND_WINDOW_MS = 5000; // Batches token deltas per (msgIdx, candIdx) and flushes them in a single // setState call per animation frame. Streaming tokens at 30-60/s × 3 candidates // otherwise causes a rerender per token. Non-token events (start/done/complete) // flush the pending deltas first to preserve ordering. // // INVARIANT: keys are message indices into the messages[] array. Callers must // ensure no message is inserted *before* a streaming message for the duration // of its stream — appending to the end is fine, mid-list insert is not. Today // every path appends to the end; if that changes, switch to a stable message // id (e.g. the placeholder's runId or a freshly-minted uuid). function useTokenBatcher( setMessages: React.Dispatch>, ) { // Lazy-init refs to avoid allocating a fresh Map on every render. const pending = useRef> | null>(null); if (pending.current === null) pending.current = new Map(); const rafId = useRef(null); const flush = useCallback(() => { rafId.current = null; const batch = pending.current; if (!batch || batch.size === 0) return; pending.current = new Map(); setMessages((prev) => prev.map((m, i) => { const perCand = batch.get(i); if (!perCand) return m; const cands = [...(m.candidates ?? [])]; for (const [ci, delta] of perCand) { if (cands[ci]) { cands[ci] = { ...cands[ci], text: cands[ci].text + delta }; } } return { ...m, candidates: cands }; }), ); }, [setMessages]); const queueToken = useCallback( (msgIdx: number, candIdx: number, delta: string) => { const batch = pending.current!; let perMsg = batch.get(msgIdx); if (!perMsg) { perMsg = new Map(); batch.set(msgIdx, perMsg); } perMsg.set(candIdx, (perMsg.get(candIdx) ?? "") + delta); if (rafId.current === null) { rafId.current = window.requestAnimationFrame(flush); } }, [flush], ); const flushNow = useCallback(() => { if (rafId.current !== null) { window.cancelAnimationFrame(rafId.current); rafId.current = null; } flush(); }, [flush]); // Cancel any pending rAF on unmount — otherwise a persona switch mid-stream // leaves a scheduled flush that calls setMessages against the new state. useEffect(() => { return () => { if (rafId.current !== null) { window.cancelAnimationFrame(rafId.current); rafId.current = null; } pending.current = null; }; }, []); return { queueToken, flushNow }; } export function ChatPanel({ userId, personaName, sensing, affectOverride, onAirTextConsumed, onHeadSignalConsumed, messages, setMessages, onLatency, backendReady, }: Props) { const [input, setInput] = useState(""); const [loading, setLoading] = useState(false); const [turnaroundLoading, setTurnaroundLoading] = useState(false); const [regenerateLoading, setRegenerateLoading] = useState(false); const { queueToken, flushNow } = useTokenBatcher(setMessages); const [voiceText, setVoiceText] = useState(null); const [voiceNote, setVoiceNote] = useState(null); const voice = useVoice(); const micAvailable = isVoiceCapable(userId) && voice.supported; const bottomRef = useRef(null); const lastResponseTsRef = useRef(0); const lastTurnIdRef = useRef(null); // turn_id of the most recent turn that was already turned around — guards // against the new turnaround bubble's own head-signal re-firing turnaround // on itself. const turnaroundConsumedTurnRef = useRef(null); const evalPollAbortsRef = useRef>(new Set()); useEffect(() => { bottomRef.current?.scrollIntoView({ behavior: "smooth" }); }, [messages]); // Reset per-turn state when the persona changes (parent clears `messages` // and resets the backend session — the frontend turn counter must follow). useEffect(() => { lastTurnIdRef.current = null; turnaroundConsumedTurnRef.current = null; lastResponseTsRef.current = 0; evalPollAbortsRef.current.forEach((ac) => ac.abort()); evalPollAbortsRef.current.clear(); setVoiceText(null); setVoiceNote(null); }, [userId]); useEffect(() => { const active = evalPollAbortsRef.current; return () => { active.forEach((ac) => ac.abort()); active.clear(); }; }, []); const startEvalPolling = useCallback( (runId: string | null | undefined) => { if (!runId) return; const ac = new AbortController(); evalPollAbortsRef.current.add(ac); void pollEvals(runId, { signal: ac.signal }) .then((scores) => { if (ac.signal.aborted || !scores) return; setMessages((prev) => prev.map((m) => m.runId === runId ? { ...m, evalScores: scores } : m ) ); }) .finally(() => { evalPollAbortsRef.current.delete(ac); }); }, [setMessages] ); const handleTurnaround = useCallback( async (reason: "head" | "manual") => { if (!userId || !backendReady || turnaroundLoading || loading) return; const targetTurnId = lastTurnIdRef.current; if (targetTurnId === null) return; if (turnaroundConsumedTurnRef.current === targetTurnId) return; turnaroundConsumedTurnRef.current = targetTurnId; setTurnaroundLoading(true); try { const res = await sendTurnaround({ user_id: userId, turn_id: targetTurnId, head_signal: reason === "head" ? sensing.headSignal : null, }); lastTurnIdRef.current = res.turn_id; turnaroundConsumedTurnRef.current = res.turn_id; setMessages((prev) => { const next = [...prev]; for (let i = next.length - 1; i >= 0; i--) { if (next[i].role === "aac_user" && !next[i].isTurnaround) { next[i] = { ...next[i], rephrased: true, picked: true }; break; } } next.push({ role: "aac_user", content: res.response, latency: res.latency, affect: res.affect, runId: res.run_id, turnId: res.turn_id, evalScores: null, isTurnaround: true, candidates: res.candidates ?? [], picked: true, }); return next; }); onLatency(res.latency); startEvalPolling(res.run_id); // Do NOT advance lastResponseTsRef — keep the original turn's window so // the user can't head-shake the turnaround itself into another loop. } catch (e) { setMessages((prev) => [ ...prev, { role: "aac_user", content: `Error rephrasing: ${ e instanceof Error ? e.message : "request failed" }`, isTurnaround: true, }, ]); } finally { if (reason === "head") onHeadSignalConsumed(); setTurnaroundLoading(false); } }, [ userId, backendReady, turnaroundLoading, loading, sensing.headSignal, setMessages, onLatency, onHeadSignalConsumed, startEvalPolling, ] ); const handleRegenerate = useCallback( async (msgIdx: number) => { if (!userId || !backendReady || regenerateLoading || loading) return; const msg = messages[msgIdx]; if (!msg || !msg.candidates || msg.picked || msg.turnId === undefined) return; const currentRound = msg.candidates; const priorRounds = msg.rejectedRounds ?? []; const rejected_texts = [ ...priorRounds.flat().map((c) => c.text), ...currentRound.map((c) => c.text), ]; setRegenerateLoading(true); // Move the current round into rejectedRounds + clear candidates so the // UI shows empty-card placeholders while streams fill in. setMessages((prev) => prev.map((m, i) => i === msgIdx ? { ...m, candidates: [], rejectedRounds: [...priorRounds, currentRound], picked: false, } : m, ), ); const updateMsg = ( updater: (m: ChatMessage) => ChatMessage, ) => { setMessages((prev) => prev.map((m, i) => (i === msgIdx ? updater(m) : m)), ); }; try { await streamRegenerate( { user_id: userId, turn_id: msg.turnId, rejected_texts, }, (evt) => { if (evt.type === "token") { queueToken(msgIdx, evt.idx, evt.delta); return; } flushNow(); if (evt.type === "candidate_start") { updateMsg((m) => { const cands = [...(m.candidates ?? [])]; while (cands.length <= evt.idx) { cands.push({ text: "", strategy: "pending", grounded_buckets: [], }); } cands[evt.idx] = { text: "", strategy: evt.strategy, grounded_buckets: evt.grounded_buckets, }; return { ...m, candidates: cands }; }); } else if (evt.type === "candidate_done") { updateMsg((m) => { const cands = [...(m.candidates ?? [])]; if (cands[evt.idx]) { cands[evt.idx] = { ...cands[evt.idx], text: evt.text }; } return { ...m, candidates: cands }; }); } else if (evt.type === "complete") { const res = evt.response; lastTurnIdRef.current = res.turn_id; updateMsg((m) => ({ ...m, content: res.response, latency: res.latency, affect: res.affect, runId: res.run_id, turnId: res.turn_id, evalScores: null, candidates: res.candidates ?? m.candidates ?? [], picked: false, })); onLatency(res.latency); startEvalPolling(res.run_id); } }, ); } catch (e) { flushNow(); console.warn("streamRegenerate failed", e); } finally { setRegenerateLoading(false); } }, [ userId, backendReady, regenerateLoading, loading, messages, setMessages, queueToken, flushNow, onLatency, startEvalPolling, ] ); useEffect(() => { if ( sensing.headSignal !== "HEAD_NOD_DISSATISFIED" && sensing.headSignal !== "HEAD_SHAKE" ) { return; } // If the most recent AAC message has an open picker, head-signal means // "regenerate" — the user hasn't committed, so there's nothing to // "rephrase" yet. let openPickerIdx = -1; for (let i = messages.length - 1; i >= 0; i--) { const m = messages[i]; if (m.role !== "aac_user") continue; if (!m.picked && (m.candidates?.length ?? 0) > 1) openPickerIdx = i; break; } if (openPickerIdx !== -1) { handleRegenerate(openPickerIdx); onHeadSignalConsumed(); return; } const targetTurnId = lastTurnIdRef.current; const eligible = targetTurnId !== null && turnaroundConsumedTurnRef.current !== targetTurnId && lastResponseTsRef.current > 0 && performance.now() - lastResponseTsRef.current <= TURNAROUND_WINDOW_MS; if (eligible) { handleTurnaround("head"); return; } // Not eligible — keep the chip visible briefly so the user can see that // detection fired, then clear it. (Instant clear made detection invisible.) const id = window.setTimeout(() => onHeadSignalConsumed(), 1500); return () => window.clearTimeout(id); }, [ sensing.headSignal, handleTurnaround, handleRegenerate, onHeadSignalConsumed, messages, ]); async function handleSend() { if (!input.trim() || !userId || !backendReady || loading) return; const query = input.trim(); setInput(""); setLoading(true); const airText = sensing.airWrittenText || null; const vText = voiceText; const resolved = resolveIntent(vText, airText); // Push the partner bubble, and a placeholder AAC message we'll fill in // progressively. We need the placeholder's index to target updates — use // a ref captured from the setter so we don't rely on stale state. let placeholderIdx = -1; setMessages((prev) => { const next = [ ...prev, { role: "partner" as const, content: query }, { role: "aac_user" as const, content: "", candidates: [] as Candidate[], picked: false, }, ]; placeholderIdx = next.length - 1; return next; }); const updatePlaceholder = ( updater: (m: ChatMessage) => ChatMessage, ) => { setMessages((prev) => prev.map((m, i) => (i === placeholderIdx ? updater(m) : m)), ); }; try { await streamChat( { user_id: userId, query, affect_override: affectOverride ?? sensing.affect, gesture_tag: sensing.gestureTag, gaze_bucket: sensing.gazeBucket, air_written_text: airText, head_signal: sensing.headSignal, voice_text: vText, resolved_intent: resolved.source === "none" ? null : resolved, }, (evt) => { if (evt.type === "token") { queueToken(placeholderIdx, evt.idx, evt.delta); return; } // Any non-token event must see the latest text — flush the queue first. flushNow(); if (evt.type === "candidate_start") { updatePlaceholder((m) => { const cands = [...(m.candidates ?? [])]; while (cands.length <= evt.idx) { cands.push({ text: "", strategy: "pending", grounded_buckets: [], }); } cands[evt.idx] = { text: "", strategy: evt.strategy, grounded_buckets: evt.grounded_buckets, }; return { ...m, candidates: cands }; }); } else if (evt.type === "candidate_done") { updatePlaceholder((m) => { const cands = [...(m.candidates ?? [])]; if (cands[evt.idx]) { cands[evt.idx] = { ...cands[evt.idx], text: evt.text }; } return { ...m, candidates: cands }; }); } else if (evt.type === "complete") { const res = evt.response; lastTurnIdRef.current = res.turn_id; updatePlaceholder((m) => ({ ...m, content: res.response, latency: res.latency, affect: res.affect, runId: res.run_id, turnId: res.turn_id, evalScores: null, candidates: res.candidates ?? m.candidates ?? [], picked: (res.candidates ?? []).length <= 1, })); onLatency(res.latency); lastResponseTsRef.current = performance.now(); startEvalPolling(res.run_id); } }, ); } catch (e) { flushNow(); updatePlaceholder((m) => ({ ...m, content: `Error: ${e instanceof Error ? e.message : "request failed"}`, })); } finally { if (airText) onAirTextConsumed(); // Clear voice state unconditionally — a failed send shouldn't silently // re-attach a stale transcript to the next turn. User can re-tap mic. setVoiceText(null); setVoiceNote(null); setLoading(false); } } const handlePick = useCallback( async (msgIdx: number, candIdx: number) => { const msg = messages[msgIdx]; if (!msg || !msg.candidates || !msg.runId || !userId) return; if (msg.picked) return; const picked = msg.candidates[candIdx]; if (!picked) return; setMessages((prev) => prev.map((m, i) => i === msgIdx ? { ...m, content: picked.text, picked: true, pickedIdx: candIdx, } : m ) ); try { await sendPick({ run_id: msg.runId, user_id: userId, picked_idx: candIdx, }); } catch (e) { console.warn("sendPick failed", e); } }, [messages, setMessages, userId] ); const handleMic = useCallback(async () => { if (!micAvailable || voice.listening) return; setVoiceNote("Listening..."); try { const cap = await voice.capture(); if (cap.transcript) { setVoiceText(cap.transcript); setVoiceNote(`Heard: "${cap.transcript}"`); } else { setVoiceNote("No speech detected."); } } catch (e) { setVoiceNote( `Mic error: ${e instanceof Error ? e.message : "failed"}` ); } }, [micAvailable, voice]); const canTurnaround = !!userId && backendReady && !loading && !turnaroundLoading && lastTurnIdRef.current !== null; return (
Talking as: {personaName || "select a persona"}
{messages.map((msg, i) => { const hasRegenerated = (msg.rejectedRounds?.length ?? 0) > 0; const showPicker = msg.role === "aac_user" && !msg.picked && !!msg.candidates && (msg.candidates.length > 1 || hasRegenerated); if (showPicker) { const priorRounds = msg.rejectedRounds ?? []; return (
AAC User pick one ({msg.candidates!.length} options) {priorRounds.map((round, ri) => (
rejected round {ri + 1}
{round.map((cand, ci) => (
{STRATEGY_LABELS[cand.strategy] ?? cand.strategy}
{cand.text}
))}
))}
{msg.candidates!.map((cand, ci) => ( ))}
); } return (
{msg.role === "partner" ? "Partner" : "AAC User"} {msg.rephrased && ( rephrased )} {msg.isTurnaround && ( ↻ turnaround )} {msg.picked && msg.pickedIdx !== undefined && msg.candidates && msg.candidates[msg.pickedIdx] && ( ✓ {STRATEGY_LABELS[msg.candidates[msg.pickedIdx].strategy] ?? msg.candidates[msg.pickedIdx].strategy} )}

{msg.content}

{msg.role === "aac_user" && msg.runId && userId && ( )}
); })} {turnaroundLoading && (
AAC User

↻ Rephrasing...

)}
{micAvailable && voiceNote && (
{voiceNote}
)}
setInput(e.target.value)} onKeyDown={(e) => e.key === "Enter" && !e.shiftKey && handleSend()} placeholder={backendReady ? "Type as the communication partner..." : "Waiting for backend to load models..."} disabled={!userId || loading || !backendReady} /> {micAvailable && ( )}
); }