diff --git a/.env.example b/.env.example index e74afb3e9cdc3d1d4b5d1d422bbe7251456a8905..7859d8bbf2cb26e490b84588fb8fa00ef57bb8c5 100644 --- a/.env.example +++ b/.env.example @@ -21,7 +21,7 @@ LOCAL_BASE_URL=http://localhost:11434/v1 LOCAL_MODEL=gemma4:31b-cloud # ── MLflow ──────────────────────────────────────────────────────────────────── -MLFLOW_TRACKING_URI=mlruns +MLFLOW_TRACKING_URI=sqlite:///mlflow.db MLFLOW_EXPERIMENT=aac-chatbot # ── Thinking mode ───────────────────────────────────────────────────────────── diff --git a/.gitignore b/.gitignore index 8d9f8d5eb7d9f5e1d515a1743372119f598a125a..338a3b3b0c8723b729f59c5d415a8a1fdaf1ec5f 100644 --- a/.gitignore +++ b/.gitignore @@ -22,13 +22,18 @@ data/faiss_store/ # Air-writing templates (large numpy files, track separately if needed) data/air_write_templates/ -# MLflow run artifacts +# MLflow mlruns/ +mlflow.db # Latency logs timings.csv *.csv +# Frontend +node_modules/ +frontend/dist/ + # IDE .vscode/ .idea/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d254ec9a9f000582a0cc7344a1f2fd06d5e42fce --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,16 @@ +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.11.12 + hooks: + - id: ruff + args: [--fix] + - id: ruff-format + + - repo: local + hooks: + - id: eslint + name: eslint + entry: bash -c 'cd frontend && npx eslint src/' + language: system + files: ^frontend/src/.*\.(ts|tsx)$ + pass_filenames: false diff --git a/CLAUDE.md b/CLAUDE.md index 2e1eef84115467aa450b8816c01333637fe6d07b..ba5fcefcd9320881c11f7ee8020ac17c28db1329 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -12,18 +12,25 @@ Orchestrated as a **LangGraph stateful directed graph** across five layers. ## Architecture ``` -main.py / api/main.py / ui/app.py - └── pipeline/graph.py ← LangGraph StateGraph (5 nodes + cond. edges) - ├── pipeline/nodes/intent.py L2 — LLM + Pydantic intent routing - ├── pipeline/nodes/retrieval.py L3 — FAISS + BGE retrieval (fast / full) - ├── pipeline/nodes/planner.py L4 — expression-conditioned generation - └── pipeline/nodes/feedback.py L5 — MLflow logging + Bayesian priors - -sensing/ L1 — MediaPipe face mesh, gesture, gaze, air writing -retrieval/ FAISS ops, HDBSCAN clustering, Bayesian bucket priors -generation/ Multi-tier LLM client (vLLM primary / fallback / Ollama local) -guardrails/ Input + output safety checks -config/ Pydantic BaseSettings — all config in one place +frontend/ React + Vite + TypeScript + src/hooks/useSensing.ts MediaPipe JS — affect, gesture, gaze, air-writing (browser-side) + src/components/ChatPanel.tsx Chat UI → POST /chat with sensing labels + +backend/ Python (conda env: aac-chatbot) + main.py CLI entry point + api/main.py FastAPI REST API + pipeline/graph.py LangGraph StateGraph (5 nodes + conditional edges) + pipeline/nodes/intent.py L2 — LLM + Pydantic intent routing + pipeline/nodes/retrieval.py L3 — FAISS + BGE retrieval (fast / full) + pipeline/nodes/planner.py L4 — expression-conditioned generation + pipeline/nodes/feedback.py L5 — MLflow logging + Bayesian priors + sensing/ L1 — MediaPipe face mesh, gesture, gaze, air writing (Python, CLI use) + retrieval/ FAISS ops, HDBSCAN clustering, Bayesian bucket priors + generation/ Multi-tier LLM client (vLLM primary / fallback / Ollama local) + guardrails/ Input + output safety checks + config/ Pydantic BaseSettings — all config in one place + +data/ Shared data (personas, FAISS indexes) ``` ## Key Design Decisions @@ -39,6 +46,8 @@ config/ Pydantic BaseSettings — all config in one place - **Expression-conditioned response shaping** — affect steers tone, retrieval depth, and candidate ranking (not just metadata annotation) - **Bayesian bucket priors** — session-level P(bucket) updated after each accepted turn +- **Browser-side sensing** — MediaPipe JS runs in React frontend, only classified + labels (affect, gesture, gaze bucket) are sent to the backend API --- @@ -57,22 +66,22 @@ config/ Pydantic BaseSettings — all config in one place ## How to Run ```bash -# One-time setup: rebuild FAISS indexes with BGE embedder -python -m retrieval.vector_store +# One-time setup +bash setup.sh -# CLI (local Ollama tier, set ACTIVE_LLM_TIER=local in .env) -python main.py --debug +# CLI (local Ollama tier) +python -m backend.main --debug # Full stack -uvicorn api.main:app --reload # FastAPI on :8000 -streamlit run ui/app.py # Streamlit on :8501 +uvicorn backend.api.main:app --reload # FastAPI on :8000 +pnpm --dir frontend dev # React on :7550 ``` --- ## Configuration -All config lives in [config/settings.py](config/settings.py) as Pydantic `BaseSettings`. +All config lives in [backend/config/settings.py](backend/config/settings.py) as Pydantic `BaseSettings`. Copy `.env.example` → `.env` and set: - `ACTIVE_LLM_TIER` — `local` (dev) | `primary` (GCP A100) | `fallback` (Qwen3-8B) @@ -95,12 +104,12 @@ Copy `.env.example` → `.env` and set: ## Development Notes - **Adding a persona**: add to `PERSONAS` in `data/generate_users.py`, re-run it, - then `python -m retrieval.vector_store` to rebuild indexes + then `python -m backend.retrieval.vector_store` to rebuild indexes - **Changing LLM**: set `ACTIVE_LLM_TIER` in `.env` — no code changes needed -- **Extending sensing**: add module under `sensing/`, wire output into - `PipelineState` fields in `pipeline/state.py` -- **Guardrail tuning**: edit signal lists in `guardrails/checks.py` -- **Affect → generation mapping**: `_AFFECT_CONFIG` in `pipeline/nodes/intent.py` - and `_PERSONA_TONE_OVERRIDES` in `pipeline/nodes/planner.py` -- The `.venv/` directory is local — do not read or modify files inside it +- **Extending sensing**: add module under `backend/sensing/`, wire output into + `PipelineState` fields in `backend/pipeline/state.py` +- **Guardrail tuning**: edit signal lists in `backend/guardrails/checks.py` +- **Affect → generation mapping**: `_AFFECT_CONFIG` in `backend/pipeline/nodes/intent.py` + and `_PERSONA_TONE_OVERRIDES` in `backend/pipeline/nodes/planner.py` - FAISS indexes in `data/faiss_store/` are gitignored — rebuilt from source JSONs +- Frontend uses pnpm, Node 22+ diff --git a/README.md b/README.md index f8d6c6ba08ef74e22d442877ccae565095f3641b..751ce630da9772be337f179d866e1d50b10fae42 100644 --- a/README.md +++ b/README.md @@ -34,16 +34,20 @@ a personalized digital twin that communicates on their behalf. ## System Architecture ``` -Webcam (L1: sensing) → Intent Decomposition (L2) → Retrieval (L3) → Generation (L4) → Feedback (L5) +React Frontend (browser) Backend (Python) + MediaPipe JS sensing ──┐ + Chat UI ───────────────┼── POST /chat ──► FastAPI ──► LangGraph Pipeline + Webcam feed ───────────┘ │ + L2 Intent ──► L3 Retrieval ──► L4 Generation ──► L5 Feedback ``` | Layer | Module | What it does | |-------|--------|-------------| -| L1 | `sensing/` | MediaPipe face mesh, hand gestures, gaze tracking, air writing | -| L2 | `pipeline/nodes/intent.py` | LLM + Pydantic-validated intent routing | -| L3 | `pipeline/nodes/retrieval.py` | FAISS + BGE embeddings + cross-encoder reranking | -| L4 | `pipeline/nodes/planner.py` | Expression-conditioned response generation (Qwen3) | -| L5 | `pipeline/nodes/feedback.py` | MLflow tracking + Bayesian bucket prior update | +| L1 | `frontend/src/hooks/useSensing.ts` | MediaPipe JS — affect, gesture, gaze, air writing (browser-side) | +| L2 | `backend/pipeline/nodes/intent.py` | LLM + Pydantic-validated intent routing | +| L3 | `backend/pipeline/nodes/retrieval.py` | FAISS + BGE embeddings + cross-encoder reranking | +| L4 | `backend/pipeline/nodes/planner.py` | Expression-conditioned response generation (Qwen3) | +| L5 | `backend/pipeline/nodes/feedback.py` | MLflow tracking + Bayesian bucket prior update | The pipeline runs as a **LangGraph stateful directed graph** with conditional edges: - FRUSTRATED affect → fast retrieval path (k=2, no reranker) @@ -53,146 +57,70 @@ The pipeline runs as a **LangGraph stateful directed graph** with conditional ed ## Prerequisites -- Python **3.10 – 3.12** (Python 3.14 has a known Pydantic v1 incompatibility warning — functional but noisy) +- Python **3.10+** (via conda) +- Node.js **22+** and **pnpm** - [Ollama](https://ollama.com) installed locally for the `local` LLM tier -- A webcam (required for the live sensing layer; optional for CLI mode) -- Git +- A webcam (for live sensing; optional for CLI mode) --- ## Setup -### 1. Clone the repository - ```bash git clone https://github.com/akashkolte/multimodal_aac_chatbot.git cd multimodal_aac_chatbot +bash setup.sh ``` -### 2. Check out the active branch - -```bash -git checkout akash/v1 -``` - -### 3. Create and activate a virtual environment - -```bash -python3 -m venv .venv -source .venv/bin/activate # macOS / Linux -# .venv\Scripts\activate # Windows -``` - -### 4. Install dependencies - -```bash -pip install -r requirements.txt -``` - -> This installs LangGraph, FAISS, sentence-transformers (BGE), FastAPI, Streamlit, MLflow, -> MediaPipe, and all other dependencies. - -### 5. Configure environment variables - -```bash -cp .env.example .env -``` - -Open `.env` and set at minimum: - -```env -ACTIVE_LLM_TIER=local # use Ollama on your machine for dev -``` - -See [Configuration](#configuration) for all options. - -### 6. Pull the local LLM model (Ollama) - -```bash -ollama pull qwen3:8b -``` - -> Make sure Ollama is running (`ollama serve`) before starting the chatbot. - -### 7. Build FAISS indexes - -The persona memory indexes must be built once with the BGE embedder before first run: - -```bash -python -m retrieval.vector_store -``` - -Expected output: -``` -Building index for arjun_mehta … Saved 25 chunks -Building index for gerald_okafor … Saved 25 chunks -Building index for mia_chen … Saved 25 chunks -All indexes built. -``` - -> You must re-run this step whenever you add or edit persona memory files. +The setup script handles: +- Conda environment creation (`aac-chatbot`, Python 3.12) +- Python dependency installation +- `.env` file creation from template +- FAISS index building (downloads BGE models on first run) +- Ollama model pull +- Frontend dependency installation (pnpm) --- ## Configuration -All settings live in [config/settings.py](config/settings.py) and can be overridden via `.env`. +All settings live in [backend/config/settings.py](backend/config/settings.py) and can be overridden via `.env`. | Variable | Default | Description | |----------|---------|-------------| | `ACTIVE_LLM_TIER` | `local` | `local` (Ollama) \| `primary` (vLLM GCP) \| `fallback` (Qwen3-8B) | | `LOCAL_MODEL` | `qwen3:8b` | Ollama model name for local dev | | `LOCAL_BASE_URL` | `http://localhost:11434/v1` | Ollama OpenAI-compatible endpoint | -| `PRIMARY_BASE_URL` | *(GCP IP)* | vLLM server URL on GCP (set when using cloud tier) | +| `PRIMARY_BASE_URL` | *(GCP IP)* | vLLM server URL on GCP | | `PRIMARY_MODEL` | `Qwen/Qwen3-30B-A3B` | Primary MoE model served via vLLM | | `FALLBACK_LATENCY_THRESHOLD` | `3.5` | Seconds before falling back to smaller model | | `MLFLOW_TRACKING_URI` | `mlruns` | Local MLflow storage path | -| `MLFLOW_EXPERIMENT` | `aac-chatbot` | MLflow experiment name | --- ## Running the Project -### Option A — CLI (simplest, no webcam needed) - -```bash -python main.py -``` - -With debug latency output: -```bash -python main.py --debug -``` +### Full stack (recommended) -Select a specific persona and LLM tier: ```bash -python main.py --user mia_chen --tier local +bash run.sh ``` -### Option B — Full stack (FastAPI + Streamlit UI) +This starts Ollama (if needed), FastAPI on `:8000`, and React on `:7550`. +Open [http://localhost:7550](http://localhost:7550) in your browser. -Start the API server in one terminal: -```bash -uvicorn api.main:app --reload --port 8000 -``` +### CLI only -Start the Streamlit frontend in another terminal: ```bash -streamlit run ui/app.py +conda activate aac-chatbot +python -m backend.main --debug ``` -Then open [http://localhost:8501](http://localhost:8501) in your browser. - -The UI includes: -- Persona selector -- Affect override controls (simulate webcam for testing) -- Live chat interface -- Per-turn latency breakdown panel - -### Option C — API only (for integration / testing) +### API only ```bash -uvicorn api.main:app --reload +conda activate aac-chatbot +uvicorn backend.api.main:app --reload ``` Example request: @@ -208,52 +136,34 @@ curl -X POST http://localhost:8000/chat \ ``` multimodal_aac_chatbot/ +├── frontend/ React + Vite + TypeScript +│ └── src/ +│ ├── components/ Chat UI, webcam, sensing status +│ ├── hooks/ useWebcam, useSensing (MediaPipe JS) +│ └── lib/ API client, sensing classification, DTW │ -├── config/ -│ └── settings.py # All config via Pydantic BaseSettings +├── backend/ Python (conda env: aac-chatbot) +│ ├── main.py CLI entry point +│ ├── api/main.py FastAPI REST API +│ ├── config/settings.py Pydantic BaseSettings +│ ├── pipeline/ +│ │ ├── graph.py LangGraph StateGraph +│ │ ├── state.py PipelineState TypedDict +│ │ └── nodes/ intent, retrieval, planner, feedback +│ ├── sensing/ MediaPipe modules (Python, CLI use) +│ ├── retrieval/ FAISS, BGE, HDBSCAN, bucket priors +│ ├── generation/llm_client.py 3-tier LLM client (vLLM / Ollama) +│ └── guardrails/checks.py Input + output safety checks │ ├── data/ -│ ├── generate_users.py # Regenerates persona memories + users.json -│ ├── users.json # Flat user index -│ ├── memories/ # Per-persona memory JSON files -│ └── faiss_store/ # Built FAISS indexes (gitignored, rebuild locally) -│ -├── sensing/ # L1 — multimodal input -│ ├── face_mesh.py # MediaPipe affect detection (MAR/EAR/BRI/LCP) -│ ├── gesture.py # Hand gesture classifier -│ ├── gaze.py # Gaze-based bucket activation (bonus) -│ └── air_writing.py # DTW air-writing stroke classifier (bonus) -│ -├── pipeline/ # LangGraph orchestration -│ ├── state.py # Typed PipelineState (TypedDict) -│ ├── graph.py # Graph definition + conditional edges -│ └── nodes/ -│ ├── intent.py # L2 — LLM + Pydantic routing -│ ├── retrieval.py # L3 — fast + full retrieval paths -│ ├── planner.py # L4 — expression-conditioned generation -│ └── feedback.py # L5 — MLflow + Bayesian prior update -│ -├── retrieval/ -│ ├── vector_store.py # FAISS ops with BGE-small-en-v1.5 -│ ├── clustering.py # HDBSCAN semantic bucketing -│ └── bucket_priors.py # Bayesian session priors -│ -├── generation/ -│ └── llm_client.py # 3-tier LLM client (vLLM / Ollama) -│ -├── guardrails/ -│ └── checks.py # Input + output safety checks -│ -├── api/ -│ └── main.py # FastAPI backend -│ -├── ui/ -│ └── app.py # Streamlit frontend +│ ├── users.json Persona index +│ ├── memories/ Per-persona memory JSONs +│ └── faiss_store/ FAISS indexes (gitignored, rebuilt) │ -├── main.py # CLI entry point -├── requirements.txt # Python dependencies -├── .env.example # Environment variable template -└── CLAUDE.md # Developer notes (AI assistant context) +├── setup.sh One-time setup script +├── run.sh Start backend + frontend +├── requirements.txt Python dependencies +└── .env.example Environment variable template ``` --- @@ -268,7 +178,7 @@ multimodal_aac_chatbot/ Each persona has 25 memory chunks across 5 buckets: `family`, `medical`, `hobbies`, `daily_routine`, `social`. -To add a new persona, edit `data/generate_users.py` and re-run `python -m retrieval.vector_store`. +To add a new persona, edit `data/generate_users.py` and re-run `python -m backend.retrieval.vector_store`. --- diff --git a/api/__init__.py b/backend/__init__.py similarity index 100% rename from api/__init__.py rename to backend/__init__.py diff --git a/generation/__init__.py b/backend/api/__init__.py similarity index 100% rename from generation/__init__.py rename to backend/api/__init__.py diff --git a/api/main.py b/backend/api/main.py similarity index 70% rename from api/main.py rename to backend/api/main.py index 5e879ad4e6cde0414d2980c2b62c3e98b0262f25..47554df8fd7ebd1c364ad48da179457a2d3dbdb3 100644 --- a/api/main.py +++ b/backend/api/main.py @@ -1,29 +1,19 @@ -""" -FastAPI backend — exposes the LangGraph pipeline as a REST API. - -Endpoints: - POST /chat — single-turn inference (non-streaming) - POST /chat/stream — streaming token delivery via SSE - GET /users — list available personas - POST /session/reset — reset session state for a user - GET /health — liveness check -""" +# FastAPI backend — REST API for the AAC pipeline. from __future__ import annotations import json -import time -from typing import AsyncGenerator from fastapi import FastAPI, HTTPException from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import StreamingResponse from pydantic import BaseModel -from config.settings import settings -from guardrails.checks import check_input -from pipeline.graph import aac_graph -from pipeline.state import PipelineState -from retrieval.bucket_priors import uniform_priors +from backend.config.settings import settings +from backend.generation.llm_client import get_client +from backend.guardrails.checks import check_input +from backend.pipeline.graph import aac_graph +from backend.pipeline.state import PipelineState +from backend.retrieval.bucket_priors import uniform_priors +from backend.retrieval.vector_store import _get_embedder, _get_reranker app = FastAPI( title="Multimodal AAC Chatbot API", @@ -38,18 +28,39 @@ app.add_middleware( allow_headers=["*"], ) +_models_ready = False + + +@app.on_event("startup") +def _warmup(): + global _models_ready + import logging + import os + + os.environ.setdefault("HF_HUB_DISABLE_TELEMETRY", "1") + logging.getLogger("sentence_transformers").setLevel(logging.WARNING) + print("Loading models...", end=" ", flush=True) + _get_embedder() + _get_reranker() + get_client() + _models_ready = True + print("ready.") + + # ── In-memory session store (replace with Redis for multi-worker deployments) ── _sessions: dict[str, dict] = {} # ── Request / response schemas ───────────────────────────────────────────────── + class ChatRequest(BaseModel): user_id: str query: str - affect_override: str | None = None # "HAPPY"|"FRUSTRATED"|"NEUTRAL"|"SURPRISED" + affect_override: str | None = None # "HAPPY"|"FRUSTRATED"|"NEUTRAL"|"SURPRISED" gesture_tag: str | None = None gaze_bucket: str | None = None + air_written_text: str | None = None class ChatResponse(BaseModel): @@ -65,10 +76,16 @@ class ChatResponse(BaseModel): # ── Helpers ──────────────────────────────────────────────────────────────────── + def _get_or_init_session(user_id: str) -> dict: if user_id not in _sessions: - with open(settings.users_json) as f: - users = {u["id"]: u for u in json.load(f)["users"]} + try: + with open(settings.users_json) as f: + users = {u["id"]: u for u in json.load(f)["users"]} + except FileNotFoundError as e: + raise HTTPException( + status_code=503, detail="users.json not found — run setup.sh" + ) from e if user_id not in users: raise HTTPException(status_code=404, detail=f"User '{user_id}' not found") _sessions[user_id] = { @@ -95,7 +112,7 @@ def _build_initial_state(req: ChatRequest, session: dict) -> PipelineState: affect=affect_state, gesture_tag=req.gesture_tag, gaze_bucket=req.gaze_bucket, - air_written_text=None, + air_written_text=req.air_written_text, raw_query=req.query, intent_route=None, generation_config=None, @@ -106,7 +123,13 @@ def _build_initial_state(req: ChatRequest, session: dict) -> PipelineState: candidates=[], selected_response=None, llm_tier_used="", - latency_log={"t_sensing": 0.0, "t_intent": 0.0, "t_retrieval": 0.0, "t_generation": 0.0, "t_total": 0.0}, + latency_log={ + "t_sensing": 0.0, + "t_intent": 0.0, + "t_retrieval": 0.0, + "t_generation": 0.0, + "t_total": 0.0, + }, mlflow_run_id=None, guardrail_passed=True, ) @@ -114,15 +137,21 @@ def _build_initial_state(req: ChatRequest, session: dict) -> PipelineState: # ── Routes ───────────────────────────────────────────────────────────────────── + @app.get("/health") def health(): - return {"status": "ok"} + return {"status": "ok", "models_ready": _models_ready} @app.get("/users") def list_users(): - with open(settings.users_json) as f: - return json.load(f) + try: + with open(settings.users_json) as f: + return json.load(f) + except FileNotFoundError as e: + raise HTTPException( + status_code=503, detail="users.json not found — run setup.sh" + ) from e @app.post("/session/reset") @@ -153,7 +182,7 @@ def chat(req: ChatRequest): # Persist updated session state session["session_history"] = result["session_history"] - session["bucket_priors"] = result["bucket_priors"] + session["bucket_priors"] = result["bucket_priors"] return ChatResponse( user_id=req.user_id, diff --git a/backend/config/__init__.py b/backend/config/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..84a6cc56d6f6eea2802f928e90f1b674135357d9 --- /dev/null +++ b/backend/config/__init__.py @@ -0,0 +1,3 @@ +from .settings import settings + +__all__ = ["settings"] diff --git a/config/settings.py b/backend/config/settings.py similarity index 61% rename from config/settings.py rename to backend/config/settings.py index 01fa5cf213261108a13e89c012c5871eab271727..6069853295624a1ffa5eda00990b53144fa3709b 100644 --- a/config/settings.py +++ b/backend/config/settings.py @@ -1,9 +1,12 @@ from pathlib import Path + from pydantic_settings import BaseSettings, SettingsConfigDict class Settings(BaseSettings): - model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore") + model_config = SettingsConfigDict( + env_file=".env", env_file_encoding="utf-8", extra="ignore" + ) # ── Paths ────────────────────────────────────────────────────────────────── data_dir: Path = Path("data") @@ -16,17 +19,18 @@ class Settings(BaseSettings): rerank_model: str = "BAAI/bge-reranker-v2-m3" retrieval_top_k: int = 5 retrieval_rerank_k: int = 3 - retrieval_fast_k: int = 2 # used when affect == FRUSTRATED + retrieval_fast_k: int = 2 # used when affect == FRUSTRATED # ── LLM tiers ───────────────────────────────────────────────────────────── # Tier 1 — primary (Qwen3-30B-A3B via vLLM on GCP) primary_model: str = "Qwen/Qwen3-30B-A3B" primary_base_url: str = "http://localhost:8000/v1" - primary_api_key: str = "token-abc" # vLLM default + primary_api_key: str = "token-abc" # vLLM default # Tier 2 — fallback dense model (Qwen3-8B via vLLM, same server) fallback_model: str = "Qwen/Qwen3-8B" fallback_base_url: str = "http://localhost:8000/v1" + fallback_api_key: str = "token-abc" # Tier 3 — local dev (Ollama on MacBook M2) local_model: str = "qwen3:8b" @@ -36,44 +40,34 @@ class Settings(BaseSettings): # Active tier: "primary" | "fallback" | "local" active_llm_tier: str = "local" - # Thinking mode: "off" = plain completion, no thinking whatsoever - # "strip" = let model think, but strip tags from output - # "full" = return raw response including blocks - # "suppress" = actively suppress thinking via /no_think (Ollama) or - # chat_template_kwargs (vLLM). Use for models like Qwen3 - # that think by default and need explicit suppression. + # off | strip | full | suppress thinking_mode: str = "off" - - # Extra token budget added on top of max_tokens when thinking is enabled - # (thinking_mode = "strip" or "full"). Set to 0 if using a non-thinking model. thinking_token_budget: int = 4096 - - # Wall-clock threshold (seconds) that triggers fallback within a turn - fallback_latency_threshold: float = 3.5 + fallback_latency_threshold: float = 3.5 # seconds before tier fallback # ── Generation ──────────────────────────────────────────────────────────── max_tokens_happy: int = 150 max_tokens_neutral: int = 100 max_tokens_frustrated: int = 60 max_tokens_surprised: int = 80 - num_candidates: int = 2 # responses generated per turn for ranking + num_candidates: int = 2 # responses generated per turn for ranking # ── Sensing ─────────────────────────────────────────────────────────────── - affect_ema_alpha: float = 0.3 # exponential moving average smoothing + affect_ema_alpha: float = 0.3 # exponential moving average smoothing gaze_dwell_threshold_s: float = 1.5 air_write_velocity_start: int = 15 # px/frame — stroke begin threshold - air_write_velocity_end: int = 5 # px/frame — stroke end threshold - air_write_end_gap_ms: int = 200 # ms of stillness to end a stroke - conflict_overlap_ms: int = 500 # audio + gesture co-occurrence window + air_write_velocity_end: int = 5 # px/frame — stroke end threshold + air_write_end_gap_ms: int = 200 # ms of stillness to end a stroke + conflict_overlap_ms: int = 500 # audio + gesture co-occurrence window # ── MLflow ──────────────────────────────────────────────────────────────── - mlflow_tracking_uri: str = "mlruns" + mlflow_tracking_uri: str = "sqlite:///mlflow.db" mlflow_experiment: str = "aac-chatbot" - # ── Candidate ranking weights (Eq. 2 in proposal) ───────────────────────── - rank_alpha: float = 0.4 # faithfulness weight - rank_beta: float = 0.3 # style similarity weight - rank_gamma: float = 0.3 # affect-match weight + # ── Candidate ranking weights ─────────────────────────────────────────────── + rank_alpha: float = 0.4 # faithfulness weight + rank_beta: float = 0.3 # style similarity weight + rank_gamma: float = 0.3 # affect-match weight settings = Settings() diff --git a/guardrails/__init__.py b/backend/generation/__init__.py similarity index 100% rename from guardrails/__init__.py rename to backend/generation/__init__.py diff --git a/generation/llm_client.py b/backend/generation/llm_client.py similarity index 51% rename from generation/llm_client.py rename to backend/generation/llm_client.py index 93bc829f0a041302bc3cfdd22f941171e52b3961..210abfb31391de60800121952d13180cd22f853f 100644 --- a/generation/llm_client.py +++ b/backend/generation/llm_client.py @@ -1,21 +1,4 @@ -""" -Multi-tier LLM client (proposal §5.6). - -All three tiers expose the same OpenAI-compatible API, so only the -base_url + model name change — no code-path differences downstream. - -Tier 1 — primary: Qwen3-30B-A3B via vLLM on GCP (A100 / T4) -Tier 2 — fallback: Qwen3-8B via vLLM on same server (latency > 3.5 s) -Tier 3 — local: Qwen3-8B via Ollama on MacBook M2 (dev / offline) - -Active tier is controlled by settings.active_llm_tier or the `tier` -argument passed explicitly by the planner node. - -Thinking mode is controlled by settings.thinking_mode: - "off" — prepend /no_think (Ollama) or chat_template_kwargs (vLLM) - "strip" — let the model think, but strip from output - "full" — return everything including blocks -""" +# Multi-tier LLM client — primary (vLLM) / fallback / local (Ollama), all OpenAI-compatible. from __future__ import annotations import re @@ -24,47 +7,41 @@ from typing import Any from openai import OpenAI -from config.settings import settings +from backend.config.settings import settings @lru_cache(maxsize=3) def _build_client(base_url: str, api_key: str) -> OpenAI: - """One cached OpenAI client per (base_url, api_key) pair.""" return OpenAI(base_url=base_url, api_key=api_key) def get_client(tier: str | None = None) -> OpenAI: - """ - Return the OpenAI-compatible client for the requested tier. - - Args: - tier: "primary" | "fallback" | "local" | None (uses settings.active_llm_tier) - """ resolved = tier or settings.active_llm_tier if resolved == "primary": return _build_client(settings.primary_base_url, settings.primary_api_key) if resolved == "fallback": - return _build_client(settings.fallback_base_url, settings.primary_api_key) + return _build_client(settings.fallback_base_url, settings.fallback_api_key) # local / default return _build_client(settings.local_base_url, settings.local_api_key) def active_model(tier: str | None = None) -> str: - """Return the model name string for the given tier.""" resolved = tier or settings.active_llm_tier - return { - "primary": settings.primary_model, + models = { + "primary": settings.primary_model, "fallback": settings.fallback_model, - "local": settings.local_model, - }[resolved] + "local": settings.local_model, + } + if resolved not in models: + raise ValueError( + f"Unknown LLM tier: '{resolved}'. Must be primary/fallback/local." + ) + return models[resolved] def _apply_no_think(messages: list[dict]) -> list[dict]: - """ - Prepend /no_think to the first user message. - This is the Ollama-compatible way to suppress thinking mode. - """ + # Prepend /no_think to first user message (Ollama thinking suppression). result = list(messages) for i, msg in enumerate(result): if msg.get("role") == "user": @@ -74,7 +51,6 @@ def _apply_no_think(messages: list[dict]) -> list[dict]: def _strip_think_tags(text: str) -> str: - """Remove blocks from model output.""" return re.sub(r".*?", "", text, flags=re.DOTALL).strip() @@ -85,17 +61,7 @@ def chat_complete( temperature: float = 0.7, **kwargs: Any, ) -> str: - """ - Model-agnostic chat completion. Returns the response text directly. - - Thinking mode behaviour is controlled entirely by settings.thinking_mode: - "off" — suppress thinking via /no_think (Ollama) or extra_body (vLLM) - "strip" — allow thinking but remove tags from the response - "full" — return the raw response including any blocks - - In local dev mode (active_llm_tier="local"), all tier requests are - redirected to Ollama — there is no separate fallback server locally. - """ + # Returns response text. Handles thinking mode and local-tier collapsing. resolved_tier = tier or settings.active_llm_tier # Local dev: no GCP server available — collapse all tiers to Ollama @@ -107,16 +73,17 @@ def chat_complete( patched_messages = messages extra_body: dict[str, Any] = kwargs.pop("extra_body", {}) - # "suppress" = actively inject /no_think or vLLM flag for models - # like Qwen3 that think by default and need explicit suppression. + # Suppress thinking for models that think by default. if settings.thinking_mode == "suppress": if resolved_tier == "local": patched_messages = _apply_no_think(messages) else: - extra_body = {**extra_body, "chat_template_kwargs": {"enable_thinking": False}} + extra_body = { + **extra_body, + "chat_template_kwargs": {"enable_thinking": False}, + } - # When thinking is enabled (strip/full), add budget so the model - # has room to reason without truncating the actual answer. + # Add thinking budget when enabled. effective_max_tokens = max_tokens if settings.thinking_mode in ("strip", "full"): effective_max_tokens = max_tokens + settings.thinking_token_budget @@ -129,7 +96,7 @@ def chat_complete( extra_body=extra_body or None, **kwargs, ) - raw = resp.choices[0].message.content or "" + raw = (resp.choices[0].message.content if resp.choices else "") or "" if settings.thinking_mode in ("off", "strip"): raw = _strip_think_tags(raw) @@ -138,7 +105,6 @@ def chat_complete( def warmup(tier: str | None = None) -> None: - """Send a minimal prompt to pre-load the model and warm KV cache.""" chat_complete( messages=[{"role": "user", "content": "hi"}], max_tokens=5, diff --git a/pipeline/__init__.py b/backend/guardrails/__init__.py similarity index 100% rename from pipeline/__init__.py rename to backend/guardrails/__init__.py diff --git a/guardrails/checks.py b/backend/guardrails/checks.py similarity index 57% rename from guardrails/checks.py rename to backend/guardrails/checks.py index 0c2a36690b8e538e4174851c953d74723deea5a4..8766b0bc15d22a4cd6ba539a7ce3a22524e4f974 100644 --- a/guardrails/checks.py +++ b/backend/guardrails/checks.py @@ -1,12 +1,4 @@ -""" -Input and output safety guardrails. - -check_input — runs BEFORE retrieval (blocks out-of-scope requests) -check_output — runs AFTER generation (catches persona breaks / hallucinations) - -Both return a result dict so the caller decides how to handle failures -rather than raising exceptions inside pipeline nodes. -""" +# Input + output safety guardrails. from __future__ import annotations # ── Signal lists ─────────────────────────────────────────────────────────────── @@ -37,50 +29,43 @@ OUT_OF_SCOPE_SIGNALS = [ ] SAFE_FALLBACK = "I don't know." -OOS_FALLBACK = "I'm here to help communicate as this person — that's a bit outside what I do." +OOS_FALLBACK = ( + "I'm here to help communicate as this person — that's a bit outside what I do." +) # ── Public API ───────────────────────────────────────────────────────────────── -def check_input(query: str) -> dict: - """ - Validate the partner's query before retrieval. - Returns: - {"allowed": bool, "reason": str | None, "fallback": str | None} - """ +def check_input(query: str) -> dict: q = query.lower().strip() if any(s in q for s in OUT_OF_SCOPE_SIGNALS): return {"allowed": False, "reason": "out_of_scope", "fallback": OOS_FALLBACK} if len(q) < 2: - return {"allowed": False, "reason": "empty_query", "fallback": "Could you repeat that?"} + return { + "allowed": False, + "reason": "empty_query", + "fallback": "Could you repeat that?", + } return {"allowed": True, "reason": None, "fallback": None} def check_output(response: str, memories: list[dict]) -> dict: - """ - Validate the generated response after generation. - - Checks: - 1. Persona break — did the model say "as an AI …"? - 2. Basic hallucination signal — response claims facts not in memories. - - Returns: - {"passed": bool, "issue": str | None, "fallback": str | None} - """ r = response.lower() if any(signal in r for signal in PERSONA_BREAK_SIGNALS): return {"passed": False, "issue": "persona_break", "fallback": SAFE_FALLBACK} - # Light hallucination check: if the model asserts specific numbers or - # proper nouns that don't appear anywhere in the retrieved memories, flag it. - # (Full NLI-based check is handled in the evaluation pipeline, not here.) + # Flag unsupported factual claims when no memories were retrieved. if not memories and _makes_factual_claim(response): - return {"passed": False, "issue": "unsupported_claim", "fallback": SAFE_FALLBACK} + return { + "passed": False, + "issue": "unsupported_claim", + "fallback": SAFE_FALLBACK, + } return {"passed": True, "issue": None, "fallback": None} @@ -88,11 +73,17 @@ def check_output(response: str, memories: list[dict]) -> dict: # ── Helpers ─────────────────────────────────────────────────────────────────── _FACTUAL_MARKERS = [ - " is ", " was ", " has ", " have ", " lives in ", - " born in ", " works at ", " studied at ", + " is ", + " was ", + " has ", + " have ", + " lives in ", + " born in ", + " works at ", + " studied at ", ] + def _makes_factual_claim(text: str) -> bool: - """Heuristic: does the text assert a specific fact?""" t = text.lower() return any(marker in t for marker in _FACTUAL_MARKERS) diff --git a/main.py b/backend/main.py similarity index 61% rename from main.py rename to backend/main.py index fa61113cdb6c3c9abbc7d4fd1e22f294f4edc842..91873000a25446919dfdde77d599ad9d07f6e5f5 100644 --- a/main.py +++ b/backend/main.py @@ -1,18 +1,4 @@ -""" -CLI entry point — thin wrapper around the LangGraph pipeline. - -Usage: - python main.py # interactive chat, local LLM tier - python main.py --user mia_chen # skip persona selection prompt - python main.py --debug # print per-turn latency table - python main.py --fast # skip LLM intent call (keyword routing), - # cuts turn time from ~2min → ~45s on M2 Mac - python main.py --tier primary # override LLM tier - -For the full UI, run the FastAPI + Streamlit stack instead: - uvicorn api.main:app --reload - streamlit run ui/app.py -""" +# CLI entry point for the AAC chatbot pipeline. from __future__ import annotations import argparse @@ -21,34 +7,45 @@ import os import sys import time -from config.settings import settings -from guardrails.checks import check_input -from pipeline.graph import aac_graph -from pipeline.state import PipelineState, GenerationConfig -from retrieval.bucket_priors import uniform_priors -from retrieval.vector_store import _get_embedder, _get_reranker +from backend.config.settings import settings +from backend.guardrails.checks import check_input +from backend.pipeline.graph import aac_graph +from backend.pipeline.state import GenerationConfig, PipelineState +from backend.retrieval.bucket_priors import uniform_priors +from backend.retrieval.vector_store import _get_embedder, _get_reranker def parse_args() -> argparse.Namespace: p = argparse.ArgumentParser(description="AAC Chatbot CLI") - p.add_argument("--user", type=str, default=None, help="Persona user_id") - p.add_argument("--debug", action="store_true", help="Print latency table each turn") - p.add_argument("--fast", action="store_true", - help="Skip LLM intent call — use keyword routing instead (faster local dev)") - p.add_argument("--tier", type=str, default=None, - choices=["primary", "fallback", "local"], - help="Override LLM tier (default: settings.active_llm_tier)") + p.add_argument("--user", type=str, default=None, help="Persona user_id") + p.add_argument("--debug", action="store_true", help="Print latency table each turn") + p.add_argument( + "--fast", + action="store_true", + help="Skip LLM intent call — use keyword routing instead (faster local dev)", + ) + p.add_argument( + "--tier", + type=str, + default=None, + choices=["primary", "fallback", "local"], + help="Override LLM tier (default: settings.active_llm_tier)", + ) return p.parse_args() # ── Fast keyword-based intent routing (bypasses the slow LLM intent call) ────── + def _keyword_intent(query: str) -> tuple[dict, GenerationConfig]: """Replicate milestone-1 keyword routing as a fast local-dev shortcut.""" q = query.lower() bucket: str | None = None - if any(w in q for w in ["medication", "medicine", "doctor", "health", "allergic", "therapy"]): + if any( + w in q + for w in ["medication", "medicine", "doctor", "health", "allergic", "therapy"] + ): bucket = "medical" elif any(w in q for w in ["family", "mom", "dad", "brother", "sister", "parents"]): bucket = "family" @@ -59,12 +56,27 @@ def _keyword_intent(query: str) -> tuple[dict, GenerationConfig]: elif any(w in q for w in ["friend", "social", "people", "party", "community"]): bucket = "social" - intent_type = "CONTEXTUAL" if any(w in q for w in ["you just said", "earlier", "you mentioned"]) else "PERSONAL" + intent_type = ( + "CONTEXTUAL" + if any(w in q for w in ["you just said", "earlier", "you mentioned"]) + else "PERSONAL" + ) route = { - "sub_intents": [{"type": intent_type, "query": query, "bucket_hint": bucket, "priority": "normal"}], - "style_constraints": {"tone_tag": "[TONE:DEFAULT]", "max_tokens": 100, - "retrieval_mode": "full", "persona_mod": "baseline"}, + "sub_intents": [ + { + "type": intent_type, + "query": query, + "bucket_hint": bucket, + "priority": "normal", + } + ], + "style_constraints": { + "tone_tag": "[TONE:DEFAULT]", + "max_tokens": 100, + "retrieval_mode": "full", + "persona_mod": "baseline", + }, "affect": "NEUTRAL", } gen_config: GenerationConfig = { @@ -92,17 +104,17 @@ def select_user(users: dict[str, dict], user_arg: str | None) -> str: print(f" {uid:20s} — {u['name']} ({u['condition']})") uid = input("\nSelect user id: ").strip() if uid not in users: - print(f"Invalid id.") + print("Invalid id.") sys.exit(1) return uid def print_latency(log: dict, turn: int) -> None: fields = ["t_sensing", "t_intent", "t_retrieval", "t_generation", "t_total"] - labels = ["sensing", "intent", "retrieval", "generation", "TOTAL"] - vals = [f"{log.get(f, 0):.3f}s" for f in fields] + labels = ["sensing", "intent", "retrieval", "generation", "TOTAL"] + vals = [f"{log.get(f, 0):.3f}s" for f in fields] widths = [max(len(l), len(v)) for l, v in zip(labels, vals)] - sep = " | " + sep = " | " print(f"\n[turn {turn} latency]") print(sep.join(l.ljust(w) for l, w in zip(labels, widths))) print(sep.join(v.ljust(w) for v, w in zip(vals, widths))) @@ -114,7 +126,6 @@ def main() -> None: # Optionally override the LLM tier at runtime if args.tier: os.environ["ACTIVE_LLM_TIER"] = args.tier - settings.active_llm_tier = args.tier users = load_users() user_id = select_user(users, args.user) @@ -152,14 +163,13 @@ def main() -> None: turn_id += 1 # --fast: resolve intent via keywords, skip the slow LLM intent node - pre_route, pre_gen_config = ( - _keyword_intent(query) if args.fast else (None, None) - ) t_intent_fast = 0.0 if args.fast: t0 = time.perf_counter() - _keyword_intent(query) # just for timing reference + pre_route, pre_gen_config = _keyword_intent(query) t_intent_fast = time.perf_counter() - t0 + else: + pre_route, pre_gen_config = None, None state = PipelineState( user_id=user_id, @@ -171,7 +181,7 @@ def main() -> None: gaze_bucket=None, air_written_text=None, raw_query=query, - intent_route=pre_route, # pre-filled → intent node sees it and skips LLM call + intent_route=pre_route, # pre-filled → intent node sees it and skips LLM call generation_config=pre_gen_config, retrieved_chunks=[], bucket_priors=bucket_priors, @@ -180,8 +190,13 @@ def main() -> None: candidates=[], selected_response=None, llm_tier_used="", - latency_log={"t_sensing": 0.0, "t_intent": round(t_intent_fast, 4), - "t_retrieval": 0.0, "t_generation": 0.0, "t_total": 0.0}, + latency_log={ + "t_sensing": 0.0, + "t_intent": round(t_intent_fast, 4), + "t_retrieval": 0.0, + "t_generation": 0.0, + "t_total": 0.0, + }, mlflow_run_id=None, guardrail_passed=True, ) @@ -191,13 +206,15 @@ def main() -> None: print(f"AAC Bot: {result['selected_response']}\n") session_history = result["session_history"] - bucket_priors = result["bucket_priors"] + bucket_priors = result["bucket_priors"] if args.debug: print_latency(result.get("latency_log") or {}, turn_id) - print(f" tier={result.get('llm_tier_used')} | " - f"retrieval={result.get('retrieval_mode_used')} | " - f"affect={(result.get('affect') or {}).get('emotion','?')}\n") + print( + f" tier={result.get('llm_tier_used')} | " + f"retrieval={result.get('retrieval_mode_used')} | " + f"affect={(result.get('affect') or {}).get('emotion', '?')}\n" + ) if __name__ == "__main__": diff --git a/pipeline/nodes/__init__.py b/backend/pipeline/__init__.py similarity index 100% rename from pipeline/nodes/__init__.py rename to backend/pipeline/__init__.py diff --git a/pipeline/graph.py b/backend/pipeline/graph.py similarity index 73% rename from pipeline/graph.py rename to backend/pipeline/graph.py index a5a84ee9045d63dd3671cb56c4ced440f4dea115..61b60a06e56c38e1e1e3e535691a377dd45afb58 100644 --- a/pipeline/graph.py +++ b/backend/pipeline/graph.py @@ -1,15 +1,8 @@ -""" -LangGraph stateful directed graph — the five-layer AAC pipeline. +# LangGraph pipeline graph — intent → retrieval → generation → feedback. +from langgraph.graph import END, StateGraph -Topology (see proposal Figure 2): - - intent ──► [affect check] ──► fast_retrieval ──► [latency check] ──► fallback_gen ──► feedback - └──► full_retrieval ──► [latency check] ──► primary_gen ──► feedback -""" -from langgraph.graph import StateGraph, END - -from pipeline.state import PipelineState -from pipeline.nodes import intent, retrieval, planner, feedback +from backend.pipeline.nodes import feedback, intent, planner, retrieval +from backend.pipeline.state import PipelineState def _route_by_affect(state: PipelineState) -> str: @@ -20,7 +13,8 @@ def _route_by_affect(state: PipelineState) -> str: def _route_by_latency(state: PipelineState) -> str: """Conditional edge: if cumulative latency > threshold, use fallback LLM.""" - from config.settings import settings + from backend.config.settings import settings + log = state.get("latency_log") or {} elapsed = log.get("t_intent", 0.0) + log.get("t_retrieval", 0.0) return "fallback" if elapsed > settings.fallback_latency_threshold else "primary" @@ -30,12 +24,12 @@ def build_graph() -> StateGraph: graph = StateGraph(PipelineState) # ── Nodes ────────────────────────────────────────────────────────────────── - graph.add_node("intent", intent.run) + graph.add_node("intent", intent.run) graph.add_node("fast_retrieval", retrieval.run_fast) graph.add_node("full_retrieval", retrieval.run_full) - graph.add_node("primary_gen", planner.run_primary) - graph.add_node("fallback_gen", planner.run_fallback) - graph.add_node("feedback", feedback.run) + graph.add_node("primary_gen", planner.run_primary) + graph.add_node("fallback_gen", planner.run_fallback) + graph.add_node("feedback", feedback.run) # ── Entry ────────────────────────────────────────────────────────────────── graph.set_entry_point("intent") @@ -60,9 +54,9 @@ def build_graph() -> StateGraph: ) # ── Feedback loop ───────────────────────────────────────────────────────── - graph.add_edge("primary_gen", "feedback") + graph.add_edge("primary_gen", "feedback") graph.add_edge("fallback_gen", "feedback") - graph.add_edge("feedback", END) + graph.add_edge("feedback", END) return graph.compile() diff --git a/retrieval/__init__.py b/backend/pipeline/nodes/__init__.py similarity index 100% rename from retrieval/__init__.py rename to backend/pipeline/nodes/__init__.py diff --git a/pipeline/nodes/feedback.py b/backend/pipeline/nodes/feedback.py similarity index 58% rename from pipeline/nodes/feedback.py rename to backend/pipeline/nodes/feedback.py index 05437d26301fd5bcb5af20942a6f3a9361dd022f..ea403f931ce07363d4d6e32718614a3a9f35d7e0 100644 --- a/pipeline/nodes/feedback.py +++ b/backend/pipeline/nodes/feedback.py @@ -1,29 +1,16 @@ -""" -L5 — Feedback Loop node. - -After a response is accepted: - 1. Log the full turn to MLflow (latency, metrics, prompt version, tier used) - 2. Update session-level Bayesian bucket priors - 3. Append the accepted turn to session history - -Rejected candidates are also logged for offline analysis. -""" +# Feedback node — MLflow logging, bucket prior update, history append. from __future__ import annotations -import json -import time - -import mlflow - -from config.settings import settings -from pipeline.state import PipelineState -from retrieval.bucket_priors import update_priors +from backend.config.settings import settings +from backend.pipeline.state import PipelineState +from backend.retrieval.bucket_priors import update_priors def run(state: PipelineState) -> dict: - t0 = time.perf_counter() - - mlflow_run_id = _log_to_mlflow(state) + try: + mlflow_run_id = _log_to_mlflow(state) + except Exception: + mlflow_run_id = None updated_priors = _update_bucket_priors(state) updated_history = _append_turn_to_history(state) @@ -36,7 +23,10 @@ def run(state: PipelineState) -> dict: # ── MLflow logging ───────────────────────────────────────────────────────────── + def _log_to_mlflow(state: PipelineState) -> str: + import mlflow + mlflow.set_tracking_uri(settings.mlflow_tracking_uri) mlflow.set_experiment(settings.mlflow_experiment) @@ -44,22 +34,26 @@ def _log_to_mlflow(state: PipelineState) -> str: affect = (state.get("affect") or {}).get("emotion", "UNKNOWN") with mlflow.start_run(run_name=f"turn-{state['turn_id']}") as run: - mlflow.log_params({ - "user_id": state["user_id"], - "turn_id": state["turn_id"], - "llm_tier": state.get("llm_tier_used", "unknown"), - "retrieval_mode": state.get("retrieval_mode_used", "unknown"), - "affect": affect, - "guardrail_passed": state.get("guardrail_passed", True), - }) - mlflow.log_metrics({ - "t_sensing": latency.get("t_sensing", 0.0), - "t_intent": latency.get("t_intent", 0.0), - "t_retrieval": latency.get("t_retrieval", 0.0), - "t_generation": latency.get("t_generation", 0.0), - "t_total": latency.get("t_total", 0.0), - "num_chunks": float(len(state.get("retrieved_chunks") or [])), - }) + mlflow.log_params( + { + "user_id": state["user_id"], + "turn_id": state["turn_id"], + "llm_tier": state.get("llm_tier_used", "unknown"), + "retrieval_mode": state.get("retrieval_mode_used", "unknown"), + "affect": affect, + "guardrail_passed": state.get("guardrail_passed", True), + } + ) + mlflow.log_metrics( + { + "t_sensing": latency.get("t_sensing", 0.0), + "t_intent": latency.get("t_intent", 0.0), + "t_retrieval": latency.get("t_retrieval", 0.0), + "t_generation": latency.get("t_generation", 0.0), + "t_total": latency.get("t_total", 0.0), + "num_chunks": float(len(state.get("retrieved_chunks") or [])), + } + ) # Log the selected response as artifact text for qualitative review mlflow.log_text( @@ -72,6 +66,7 @@ def _log_to_mlflow(state: PipelineState) -> str: # ── Bayesian bucket prior update ─────────────────────────────────────────────── + def _update_bucket_priors(state: PipelineState) -> dict[str, float]: chunks = state.get("retrieved_chunks") or [] if not chunks: @@ -90,9 +85,9 @@ def _update_bucket_priors(state: PipelineState) -> dict[str, float]: # ── Session history append ───────────────────────────────────────────────────── + def _append_turn_to_history(state: PipelineState) -> list[dict]: - """Returns a single-element list; LangGraph's Annotated[list, add] merges it.""" return [ - {"role": "partner", "content": state["raw_query"]}, + {"role": "partner", "content": state["raw_query"]}, {"role": "aac_user", "content": state.get("selected_response") or ""}, ] diff --git a/pipeline/nodes/intent.py b/backend/pipeline/nodes/intent.py similarity index 73% rename from pipeline/nodes/intent.py rename to backend/pipeline/nodes/intent.py index d412c072ee1b50bf3a3297641f0bff4ad4187457..3546d5a5b12298d7741eaecd0fbe9ef48b98675b 100644 --- a/pipeline/nodes/intent.py +++ b/backend/pipeline/nodes/intent.py @@ -1,20 +1,15 @@ -""" -L2 — Agentic Intent Decomposition node. - -Receives the partner query + affect state, calls the controller LLM once -(non-thinking mode, ReAct style), and returns a Pydantic-validated -IntentRoute that drives all downstream routing decisions. -""" +# Intent decomposition node — LLM-based query classification and routing. from __future__ import annotations import re import time -from typing import Literal, Optional +from typing import Literal from pydantic import BaseModel -from config.settings import settings -from generation.llm_client import chat_complete -from pipeline.state import PipelineState, GenerationConfig, IntentRoute + +from backend.config.settings import settings +from backend.generation.llm_client import chat_complete +from backend.pipeline.state import GenerationConfig, IntentRoute, PipelineState # ── Pydantic output schemas ──────────────────────────────────────────────────── @@ -25,15 +20,17 @@ AffectEmotion = Literal["HAPPY", "FRUSTRATED", "NEUTRAL", "SURPRISED"] class SubIntentSchema(BaseModel): type: Literal["PERSONAL", "CONTEXTUAL", "OPEN_DOMAIN"] query: str - bucket_hint: Optional[BucketType] = None + bucket_hint: BucketType | None = None priority: Literal["fast", "normal"] = "normal" class StyleConfig(BaseModel): - tone_tag: str # e.g. "[TONE:WITTY_SARCASTIC]" + tone_tag: str # e.g. "[TONE:WITTY_SARCASTIC]" max_tokens: int - retrieval_mode: str # "fast" | "full" - persona_mod: str # "amplify_quirks" | "suppress_humor" | "baseline" | "add_confirmation" + retrieval_mode: str # "fast" | "full" + persona_mod: ( + str # "amplify_quirks" | "suppress_humor" | "baseline" | "add_confirmation" + ) class IntentRouteSchema(BaseModel): @@ -42,7 +39,7 @@ class IntentRouteSchema(BaseModel): affect: AffectEmotion -# ── Affect → generation config mapping (proposal Table 1) ───────────────────── +# ── Affect → generation config mapping ──────────────────────────────────────── _AFFECT_CONFIG: dict[str, GenerationConfig] = { "HAPPY": { @@ -91,24 +88,30 @@ Respond ONLY with valid JSON matching the IntentRoute schema. No extra text. """ -def _build_user_prompt(query: str, affect: str, persona_name: str) -> str: +def _build_user_prompt( + query: str, affect: str, persona_name: str, air_written_text: str | None = None +) -> str: + air_note = ( + f'\nAir-written supplement: "{air_written_text}"' if air_written_text else "" + ) return ( f"Persona: {persona_name}\n" f"Affect: {affect}\n" - f"Partner query: {query}\n\n" + f"Partner query: {query}{air_note}\n\n" "Produce the IntentRoute JSON:" ) # ── Node entry point ─────────────────────────────────────────────────────────── + def run(state: PipelineState) -> dict: """LangGraph node: intent decomposition.""" t0 = time.perf_counter() # --fast mode: intent_route already resolved by keyword routing in main.py if state.get("intent_route") and state.get("generation_config"): - return {} # nothing to update — downstream nodes use the pre-filled values + return {} # nothing to update — downstream nodes use the pre-filled values affect_state = state.get("affect") or {} emotion: str = affect_state.get("emotion", "NEUTRAL") @@ -123,10 +126,23 @@ def run(state: PipelineState) -> dict: for attempt in range(3): # LangGraph retry logic (up to 2 retries) messages = [ {"role": "system", "content": _SYSTEM_PROMPT}, - {"role": "user", "content": _build_user_prompt(query, emotion, persona_name)}, + { + "role": "user", + "content": _build_user_prompt( + query, + emotion, + persona_name, + air_written_text=state.get("air_written_text"), + ), + }, ] if attempt > 0: - messages.append({"role": "user", "content": f"Validation error: {last_error}. Fix and retry."}) + messages.append( + { + "role": "user", + "content": f"Validation error: {last_error}. Fix and retry.", + } + ) raw = chat_complete( messages=messages, @@ -151,7 +167,14 @@ def run(state: PipelineState) -> dict: if route is None: # Hard fallback: treat as a single PERSONAL intent, full retrieval route = { - "sub_intents": [{"type": "PERSONAL", "query": query, "bucket_hint": None, "priority": "normal"}], + "sub_intents": [ + { + "type": "PERSONAL", + "query": query, + "bucket_hint": None, + "priority": "normal", + } + ], "style_constraints": gen_config, "affect": emotion, } @@ -166,5 +189,3 @@ def run(state: PipelineState) -> dict: "generation_config": gen_config, "latency_log": latency_log, } - - diff --git a/pipeline/nodes/planner.py b/backend/pipeline/nodes/planner.py similarity index 54% rename from pipeline/nodes/planner.py rename to backend/pipeline/nodes/planner.py index 3aee278be24f40d73489adb7db06561d33a81e53..23589dbdbd7d6e9e2bd85eb1d93a49c6e1173751 100644 --- a/pipeline/nodes/planner.py +++ b/backend/pipeline/nodes/planner.py @@ -1,39 +1,28 @@ -""" -L4 — Dialogue Planning & Generation node. - -Expression-conditioned response shaping (proposal §5.5): - 1. Build augmented prompt (persona profile + retrieved evidence + affect config + style exemplar) - 2. Generate N candidate responses - 3. Rank candidates by composite score: α·faithful + β·style + γ·affect_match - 4. Return the top-ranked response - -Two entry points: - run_primary — Qwen3-30B-A3B (or configured primary tier) - run_fallback — Qwen3-8B (faster, triggered by latency threshold) -""" +# Planner node — prompt building, candidate generation, composite ranking. from __future__ import annotations import time -from config.settings import settings -from generation.llm_client import chat_complete -from guardrails.checks import check_output -from pipeline.state import PipelineState +from backend.config.settings import settings +from backend.generation.llm_client import chat_complete +from backend.guardrails.checks import check_output +from backend.pipeline.state import PipelineState +from backend.sensing.gesture import GESTURE_TO_TAG # ── Persona-specific tone tags (applied on top of affect base tag) ───────────── _PERSONA_TONE_OVERRIDES: dict[str, dict[str, str]] = { "mia_chen": { - "HAPPY": "[TONE:WITTY_SARCASTIC]", - "FRUSTRATED": "[TONE:DIRECT_EMPATHETIC]", + "HAPPY": "[TONE:WITTY_SARCASTIC]", + "FRUSTRATED": "[TONE:DIRECT_EMPATHETIC]", }, "gerald_okafor": { - "HAPPY": "[TONE:WARM_FORMAL]", - "FRUSTRATED": "[TONE:MEASURED_EMPATHETIC]", + "HAPPY": "[TONE:WARM_FORMAL]", + "FRUSTRATED": "[TONE:MEASURED_EMPATHETIC]", }, "arjun_mehta": { - "HAPPY": "[TONE:DIRECT_WARM]", - "FRUSTRATED": "[TONE:MINIMAL_DIRECT]", + "HAPPY": "[TONE:DIRECT_WARM]", + "FRUSTRATED": "[TONE:MINIMAL_DIRECT]", }, } @@ -46,15 +35,9 @@ def run_fallback(state: PipelineState) -> dict: return _run(state, tier="fallback") -def route_by_latency(state: PipelineState) -> str: - """Conditional edge after retrieval nodes.""" - log = state.get("latency_log") or {} - elapsed = log.get("t_intent", 0.0) + log.get("t_retrieval", 0.0) - return "fallback" if elapsed > settings.fallback_latency_threshold else "primary" - - # ── Core implementation ──────────────────────────────────────────────────────── + def _run(state: PipelineState, tier: str) -> dict: t0 = time.perf_counter() @@ -63,22 +46,37 @@ def _run(state: PipelineState, tier: str) -> dict: affect = (state.get("affect") or {}).get("emotion", "NEUTRAL") gen_cfg = state.get("generation_config") or {} chunks = state.get("retrieved_chunks") or [] - history = (state.get("session_history") or [])[-3:] # last 3 turns only + history = (state.get("session_history") or [])[-3:] # last 3 turns only - tone_tag = _resolve_tone_tag(user_id, affect, gen_cfg.get("tone_tag", "[TONE:DEFAULT]")) - prompt = _build_prompt(profile, chunks, history, state["raw_query"], tone_tag, gen_cfg) + tone_tag = _resolve_tone_tag( + user_id, affect, gen_cfg.get("tone_tag", "[TONE:DEFAULT]") + ) + gesture_tag = state.get("gesture_tag") + air_written_text = state.get("air_written_text") + prompt = _build_prompt( + profile, + chunks, + history, + state["raw_query"], + tone_tag, + gen_cfg, + gesture_tag=gesture_tag, + air_written_text=air_written_text, + ) candidates: list[str] = [] for _ in range(settings.num_candidates): text = chat_complete( messages=[{"role": "user", "content": prompt}], - max_tokens=gen_cfg.get("max_tokens", settings.max_tokens_neutral) + 256, + max_tokens=gen_cfg.get("max_tokens", settings.max_tokens_neutral), temperature=0.7, tier=tier, ) candidates.append(text) - selected = _rank_candidates(candidates, chunks, affect, profile) + selected = _rank_candidates( + candidates, chunks, affect, profile, gesture_tag=gesture_tag + ) # Guardrail — replace with safe fallback if output breaks persona guard = check_output(selected, chunks) @@ -117,23 +115,40 @@ def _build_prompt( query: str, tone_tag: str, gen_cfg: dict, + gesture_tag: str | None = None, + air_written_text: str | None = None, ) -> str: - memory_block = "\n".join(f" [{c['bucket']}] {c['text']}" for c in chunks) or " (no memories retrieved)" - history_block = "\n".join(f" {h.get('role','?')}: {h.get('content','')}" for h in history) or " (start of session)" + memory_block = ( + "\n".join(f" [{c['bucket']}] {c['text']}" for c in chunks) + or " (no memories retrieved)" + ) + history_block = ( + "\n".join(f" {h.get('role', '?')}: {h.get('content', '')}" for h in history) + or " (start of session)" + ) style_exemplar = profile.get("style_exemplar", "") + gesture_line = "" + if gesture_tag: + g_tag = GESTURE_TO_TAG.get(gesture_tag, f"[GESTURE:{gesture_tag}]") + gesture_line = f"\nActive gesture signal: {g_tag}" + + air_writing_line = "" + if air_written_text: + air_writing_line = f'\nThe user air-wrote: "{air_written_text}" — treat as supplementary intent.' + persona_mod = gen_cfg.get("persona_mod", "baseline") persona_instruction = { - "amplify_quirks": "Amplify your characteristic style and personality.", - "suppress_humor": "Be direct and supportive. Suppress humor.", - "baseline": "Use your natural communication style.", - "add_confirmation": "Add a clarifying question or confirmation at the end.", + "amplify_quirks": "Amplify your characteristic style and personality.", + "suppress_humor": "Be direct and supportive. Suppress humor.", + "baseline": "Use your natural communication style.", + "add_confirmation": "Add a clarifying question or confirmation at the end.", }.get(persona_mod, "Use your natural communication style.") return f"""\ -You are {profile['name']}, an AAC device user with {profile['condition']}. -Communication style: {profile['style']} -{tone_tag} +You are {profile["name"]}, an AAC device user with {profile["condition"]}. +Communication style: {profile["style"]} +{tone_tag}{gesture_line}{air_writing_line} Style exemplar — match this register: {style_exemplar} @@ -147,7 +162,7 @@ Recent conversation: Partner says: {query} Instructions: -- Speak in first person as {profile['name']}. +- Speak in first person as {profile["name"]}. - {persona_instruction} - Keep response to 1-3 sentences. - If the answer isn't in your memories, say "I don't know." @@ -161,11 +176,8 @@ def _rank_candidates( chunks: list[dict], affect: str, profile: dict, + gesture_tag: str | None = None, ) -> str: - """ - Composite ranking: score = α·faithful + β·style + γ·affect_match - Simple heuristic version — replace with NLI + cosine similarity for final eval. - """ if not candidates: return "I don't know." if len(candidates) == 1: @@ -175,21 +187,29 @@ def _rank_candidates( style_words = set(profile.get("style", "").lower().split()) affect_positive_map = { - "HAPPY": ["great", "love", "enjoy", "happy", "fun"], + "HAPPY": ["great", "love", "enjoy", "happy", "fun"], "FRUSTRATED": ["okay", "fine", "sure", "yes", "no"], - "NEUTRAL": [], + "NEUTRAL": [], "SURPRISED": ["really", "oh", "interesting", "wow"], } - affect_words = set(affect_positive_map.get(affect, [])) + gesture_word_map = { + "THUMBS_UP": ["yes", "good", "agree", "great", "sure"], + "THUMBS_DOWN": ["no", "disagree", "stop", "don't"], + "POINTING": ["that", "this", "there", "see"], + "WAVING": ["hello", "hi", "bye", "goodbye"], + } + affect_words = set(affect_positive_map.get(affect, [])) | set( + gesture_word_map.get(gesture_tag or "", []) + ) def score(c: str) -> float: words = set(c.lower().split()) - faithful = len(words & evidence_words) / max(len(words), 1) - style_sim = len(words & style_words) / max(len(words), 1) - affect_m = len(words & affect_words) / max(len(words), 1) + faithful = len(words & evidence_words) / max(len(words), 1) + style_sim = len(words & style_words) / max(len(words), 1) + affect_m = len(words & affect_words) / max(len(words), 1) return ( settings.rank_alpha * faithful - + settings.rank_beta * style_sim + + settings.rank_beta * style_sim + settings.rank_gamma * affect_m ) diff --git a/pipeline/nodes/retrieval.py b/backend/pipeline/nodes/retrieval.py similarity index 68% rename from pipeline/nodes/retrieval.py rename to backend/pipeline/nodes/retrieval.py index 6ca39e74636ac90bf02388d1dcef44e8d5957b3e..ee8870a93fcf6d6f9a9e59e5c6bf72440703198c 100644 --- a/pipeline/nodes/retrieval.py +++ b/backend/pipeline/nodes/retrieval.py @@ -1,27 +1,25 @@ -""" -L3 — Semantic Bucketing & Retrieval node. - -Two entry points: - run_fast — FRUSTRATED affect: k=2, single bucket, no reranking - run_full — standard: k=5, optional bucket hint, BGE cross-encoder reranking - -Also exports the conditional edge function used by graph.py. -""" +# Retrieval node — run_fast (FRUSTRATED) and run_full paths. from __future__ import annotations import time -from config.settings import settings -from pipeline.state import PipelineState, RetrievedChunk -from retrieval.vector_store import retrieve -from retrieval.bucket_priors import update_priors +from backend.config.settings import settings +from backend.pipeline.state import PipelineState, RetrievedChunk +from backend.retrieval.vector_store import retrieve def run_fast(state: PipelineState) -> dict: """Fast retrieval path for FRUSTRATED affect (k=2, no reranker).""" t0 = time.perf_counter() - bucket_hint = _top_prior_bucket(state["bucket_priors"]) + priors = state["bucket_priors"] + prior_vals = list(priors.values()) if priors else [] + priors_uniform = prior_vals and (max(prior_vals) - min(prior_vals)) < 0.05 + bucket_hint = ( + state.get("gaze_bucket") + if priors_uniform and state.get("gaze_bucket") + else _top_prior_bucket(priors) + ) chunks = retrieve( query=state["raw_query"], user_id=state["user_id"], @@ -41,9 +39,8 @@ def run_full(state: PipelineState) -> dict: # Prefer gaze hint > intent bucket hint > None route = state.get("intent_route") or {} sub_intents = route.get("sub_intents", []) - bucket_hint = ( - state.get("gaze_bucket") - or next((si.get("bucket_hint") for si in sub_intents if si.get("bucket_hint")), None) + bucket_hint = state.get("gaze_bucket") or next( + (si.get("bucket_hint") for si in sub_intents if si.get("bucket_hint")), None ) chunks = retrieve( @@ -58,14 +55,9 @@ def run_full(state: PipelineState) -> dict: return _build_return(state, chunks, "full", t0) -def route_by_affect(state: PipelineState) -> str: - """Conditional edge function — called by graph.py after the intent node.""" - emotion = (state.get("affect") or {}).get("emotion", "NEUTRAL") - return "fast" if emotion == "FRUSTRATED" else "full" - - # ── Helpers ─────────────────────────────────────────────────────────────────── + def _top_prior_bucket(priors: dict[str, float]) -> str | None: if not priors: return None diff --git a/pipeline/state.py b/backend/pipeline/state.py similarity index 53% rename from pipeline/state.py rename to backend/pipeline/state.py index dba2a1417e1dc9340ac8f73edeba76667fa1b7c3..7786256fb90fa3d37ccff03293ac66a2c75e3fb1 100644 --- a/pipeline/state.py +++ b/backend/pipeline/state.py @@ -1,56 +1,54 @@ -""" -Typed state object that flows through every LangGraph node. - -Each node receives the full PipelineState and returns a dict -containing only the keys it updates — LangGraph merges them. -""" +# Typed state flowing through every LangGraph node. from __future__ import annotations -from typing import Annotated, Any, Optional -from typing_extensions import TypedDict import operator +from typing import Annotated, Any +from typing_extensions import TypedDict # ── Sub-types ────────────────────────────────────────────────────────────────── + class AffectVector(TypedDict): - MAR: float # Mouth Aspect Ratio - EAR: float # Eye Aspect Ratio - BRI: float # Brow Raise Index - LCP: float # Lip Corner Pull + MAR: float # Mouth Aspect Ratio + EAR: float # Eye Aspect Ratio + BRI: float # Brow Raise Index + LCP: float # Lip Corner Pull class AffectState(TypedDict): - emotion: str # "HAPPY" | "FRUSTRATED" | "NEUTRAL" | "SURPRISED" + emotion: str # "HAPPY" | "FRUSTRATED" | "NEUTRAL" | "SURPRISED" vector: AffectVector smoothed: AffectVector # EMA-smoothed vector class RetrievedChunk(TypedDict): text: str - bucket: str # family | medical | hobbies | daily_routine | social + bucket: str # family | medical | hobbies | daily_routine | social user: str - score: float # cross-encoder rerank score + score: float # cross-encoder rerank score class SubIntent(TypedDict): - type: str # "PERSONAL" | "CONTEXTUAL" | "OPEN_DOMAIN" + type: str # "PERSONAL" | "CONTEXTUAL" | "OPEN_DOMAIN" query: str - bucket_hint: Optional[str] - priority: str # "fast" | "normal" + bucket_hint: str | None + priority: str # "fast" | "normal" class IntentRoute(TypedDict): sub_intents: list[SubIntent] - style_constraints: dict[str, Any] # tone, max_tokens, etc. + style_constraints: dict[str, Any] # tone, max_tokens, etc. affect: str class GenerationConfig(TypedDict): max_tokens: int - tone_tag: str # e.g. "[TONE:WITTY_SARCASTIC]" - retrieval_mode: str # "fast" | "full" - persona_mod: str # "amplify_quirks" | "suppress_humor" | "baseline" | "add_confirmation" + tone_tag: str # e.g. "[TONE:WITTY_SARCASTIC]" + retrieval_mode: str # "fast" | "full" + persona_mod: ( + str # "amplify_quirks" | "suppress_humor" | "baseline" | "add_confirmation" + ) class LatencyLog(TypedDict): @@ -63,36 +61,37 @@ class LatencyLog(TypedDict): # ── Main pipeline state ──────────────────────────────────────────────────────── + class PipelineState(TypedDict): # ── Session context (set at turn start, stable across nodes) ────────────── user_id: str - persona_profile: dict[str, Any] # full profile from users.json + persona_profile: dict[str, Any] # full profile from users.json session_history: Annotated[list[dict], operator.add] # auto-appended turn_id: int # ── L1: Sensing outputs ─────────────────────────────────────────────────── - affect: Optional[AffectState] - gesture_tag: Optional[str] # e.g. "THUMBS_UP" - gaze_bucket: Optional[str] # bucket hinted by gaze fixation - air_written_text: Optional[str] # concatenated air-written chars + affect: AffectState | None + gesture_tag: str | None # e.g. "THUMBS_UP" + gaze_bucket: str | None # bucket hinted by gaze fixation + air_written_text: str | None # concatenated air-written chars # ── L2: Intent decomposition outputs ───────────────────────────────────── - raw_query: str # partner's typed/spoken query - intent_route: Optional[IntentRoute] # Pydantic-validated routing - generation_config: Optional[GenerationConfig] + raw_query: str # partner's typed/spoken query + intent_route: IntentRoute | None # Pydantic-validated routing + generation_config: GenerationConfig | None # ── L3: Retrieval outputs ───────────────────────────────────────────────── retrieved_chunks: list[RetrievedChunk] - bucket_priors: dict[str, float] # session-level Bayesian priors - retrieval_mode_used: str # "fast" | "full" + bucket_priors: dict[str, float] # session-level Bayesian priors + retrieval_mode_used: str # "fast" | "full" # ── L4: Generation outputs ──────────────────────────────────────────────── - augmented_prompt: Optional[str] - candidates: list[str] # 2-3 candidate responses - selected_response: Optional[str] - llm_tier_used: str # "primary" | "fallback" | "local" + augmented_prompt: str | None + candidates: list[str] # 2-3 candidate responses + selected_response: str | None + llm_tier_used: str # "primary" | "fallback" | "local" # ── L5: Feedback / tracking ─────────────────────────────────────────────── - latency_log: Optional[LatencyLog] - mlflow_run_id: Optional[str] + latency_log: LatencyLog | None + mlflow_run_id: str | None guardrail_passed: bool diff --git a/sensing/__init__.py b/backend/retrieval/__init__.py similarity index 100% rename from sensing/__init__.py rename to backend/retrieval/__init__.py diff --git a/backend/retrieval/bucket_priors.py b/backend/retrieval/bucket_priors.py new file mode 100644 index 0000000000000000000000000000000000000000..66fa4a3411d8d743a867c71c8cde9a1167e186c5 --- /dev/null +++ b/backend/retrieval/bucket_priors.py @@ -0,0 +1,31 @@ +# Session-level Bayesian bucket priors — updated after each accepted turn. +from __future__ import annotations + +BUCKETS = ["family", "medical", "hobbies", "daily_routine", "social"] + + +def uniform_priors() -> dict[str, float]: + p = 1.0 / len(BUCKETS) + return {b: p for b in BUCKETS} + + +def update_priors( + priors: dict[str, float], + accepted_bucket: str, + smoothing: float = 0.1, +) -> dict[str, float]: + # Boost accepted bucket, normalise. + if not priors: + priors = uniform_priors() + + updated = {b: v + smoothing for b, v in priors.items()} + updated[accepted_bucket] = updated.get(accepted_bucket, smoothing) + 1.0 + + total = sum(updated.values()) + return {b: round(v / total, 6) for b, v in updated.items()} + + +def top_bucket(priors: dict[str, float]) -> str: + if not priors: + return BUCKETS[0] + return max(priors, key=priors.get) diff --git a/retrieval/clustering.py b/backend/retrieval/clustering.py similarity index 65% rename from retrieval/clustering.py rename to backend/retrieval/clustering.py index 0f97b6f4d9ae2bf75ae77f0b2d3abe6e7c92f477..b7655cacfaebdf75caf28e8f705d8c52fb461bee 100644 --- a/retrieval/clustering.py +++ b/backend/retrieval/clustering.py @@ -1,42 +1,19 @@ -""" -HDBSCAN-based semantic bucketing over BGE embeddings. - -Used to validate / discover thematic clusters in persona memories, -and to auto-assign bucket labels when adding new memory chunks. -The hand-authored bucket labels in the JSON files remain the ground -truth — this module provides a data-driven cross-check and supports -future expansion to unlabelled memory stores. -""" +# HDBSCAN-based semantic bucketing over BGE embeddings. from __future__ import annotations import json -from pathlib import Path import numpy as np -from config.settings import settings -from retrieval.vector_store import _get_embedder - -try: - import hdbscan - _HDBSCAN_AVAILABLE = True -except ImportError: - _HDBSCAN_AVAILABLE = False - print("[clustering] hdbscan not installed — clustering unavailable.") - +from backend.config.settings import settings +from backend.retrieval.vector_store import _get_embedder BUCKET_LABELS = ["family", "medical", "hobbies", "daily_routine", "social"] def cluster_persona_memories(user_id: str) -> dict[str, list[str]]: - """ - Embed all memory chunks for a persona and cluster with HDBSCAN. - - Returns a dict mapping cluster_id → list of memory texts. - Cluster -1 = noise (unclustered points). - """ - if not _HDBSCAN_AVAILABLE: - raise RuntimeError("hdbscan package is required. Run: pip install hdbscan") + # Embed all memory chunks for a persona and cluster with HDBSCAN. + import hdbscan memory_path = settings.memories_dir / f"{user_id}.json" with open(memory_path) as f: @@ -59,7 +36,7 @@ def cluster_persona_memories(user_id: str) -> dict[str, list[str]]: labels = clusterer.fit_predict(vecs) clusters: dict[str, list[str]] = {} - for text, label, true_bucket in zip(texts, labels, true_buckets): + for text, label, _true_bucket in zip(texts, labels, true_buckets): key = f"cluster_{label}" if label >= 0 else "noise" clusters.setdefault(key, []).append(text) @@ -67,12 +44,8 @@ def cluster_persona_memories(user_id: str) -> dict[str, list[str]]: def evaluate_bucket_alignment(user_id: str) -> dict: - """ - Compare HDBSCAN cluster assignments against hand-authored bucket labels. - Returns per-bucket purity scores (fraction of dominant label in each cluster). - """ - if not _HDBSCAN_AVAILABLE: - raise RuntimeError("hdbscan package is required.") + # Compare HDBSCAN clusters against hand-authored bucket labels, return purity scores. + import hdbscan memory_path = settings.memories_dir / f"{user_id}.json" with open(memory_path) as f: diff --git a/retrieval/vector_store.py b/backend/retrieval/vector_store.py similarity index 56% rename from retrieval/vector_store.py rename to backend/retrieval/vector_store.py index 5fcee1ee98e43f0837a19e6f1b2ceea03670cea2..686fb907c2f2b72e2bfd81ed43c171b38b4e7179 100644 --- a/retrieval/vector_store.py +++ b/backend/retrieval/vector_store.py @@ -1,44 +1,45 @@ -""" -FAISS-backed dense retrieval with BGE embeddings and cross-encoder reranking. - -Models are lazy-loaded on first use (safe for FastAPI / LangGraph workers). - -NOTE: The FAISS indexes in data/faiss_store/ must be built with BGE embeddings. - Run `python -m retrieval.vector_store` to rebuild all persona indexes. -""" +# FAISS retrieval with BGE embeddings and cross-encoder reranking. from __future__ import annotations import json -import time from functools import lru_cache from pathlib import Path -import faiss import numpy as np -from sentence_transformers import CrossEncoder, SentenceTransformer -from config.settings import settings -from pipeline.state import RetrievedChunk +from backend.config.settings import settings +from backend.pipeline.state import RetrievedChunk -# ── Lazy model singletons ────────────────────────────────────────────────────── @lru_cache(maxsize=1) -def _get_embedder() -> SentenceTransformer: +def _get_embedder(): + from sentence_transformers import SentenceTransformer + return SentenceTransformer(settings.embed_model) @lru_cache(maxsize=1) -def _get_reranker() -> CrossEncoder: +def _get_reranker(): + from sentence_transformers import CrossEncoder + return CrossEncoder(settings.rerank_model) +@lru_cache(maxsize=1) +def _get_faiss(): + import faiss + + return faiss + + # ── Index cache (one FAISS index per user_id) ───────────────────────────────── -_index_cache: dict[str, tuple[faiss.Index, list[dict]]] = {} +_index_cache: dict[str, tuple] = {} -def load_index(user_id: str) -> tuple[faiss.Index, list[dict]]: +def load_index(user_id: str): if user_id not in _index_cache: + faiss = _get_faiss() store_path = settings.faiss_store_dir / user_id index = faiss.read_index(str(store_path / "index.faiss")) with open(store_path / "meta.json") as f: @@ -49,6 +50,7 @@ def load_index(user_id: str) -> tuple[faiss.Index, list[dict]]: # ── Core retrieve function ───────────────────────────────────────────────────── + def retrieve( query: str, user_id: str, @@ -56,67 +58,45 @@ def retrieve( rerank_k: int = 3, bucket_filter: str | None = None, use_reranker: bool = True, - debug: bool = False, ) -> list[RetrievedChunk]: - """ - Two-stage retrieval: - 1. BGE-small-en-v1.5 bi-encoder → FAISS IndexFlatIP (cosine similarity) - 2. BGE-reranker-v2-m3 cross-encoder reranking (multilingual, skippable) - - Args: - query: Partner's text query. - user_id: Persona identifier (e.g. "mia_chen"). - top_k: Number of candidates from FAISS before reranking. - rerank_k: Final number of chunks returned after reranking. - bucket_filter: If set, restrict candidates to this memory bucket. - use_reranker: False for the FRUSTRATED fast path. - debug: Return timing breakdown alongside results. - """ embedder = _get_embedder() index, meta = load_index(user_id) - t0 = time.perf_counter() - q_vec = embedder.encode( - [query], convert_to_numpy=True, normalize_embeddings=True - ) - t_embed = time.perf_counter() - t0 - - t0 = time.perf_counter() + q_vec = embedder.encode([query], convert_to_numpy=True, normalize_embeddings=True) _, idxs = index.search(q_vec, top_k) - t_faiss = time.perf_counter() - t0 - candidates = [meta[i] for i in idxs[0] if i < len(meta)] + candidates = [meta[i] for i in idxs[0] if 0 <= i < len(meta)] if bucket_filter: filtered = [c for c in candidates if c["bucket"] == bucket_filter] - candidates = filtered if filtered else candidates # fallback: all buckets + candidates = filtered if filtered else candidates # fallback: all buckets - t0 = time.perf_counter() if use_reranker and len(candidates) > 1: reranker = _get_reranker() pairs = [(query, c["text"]) for c in candidates] ce_scores = reranker.predict(pairs) ranked = sorted(zip(ce_scores, candidates), key=lambda x: x[0], reverse=True) top = [ - RetrievedChunk(text=c["text"], bucket=c["bucket"], user=c["user"], score=float(s)) + RetrievedChunk( + text=c["text"], bucket=c["bucket"], user=c["user"], score=float(s) + ) for s, c in ranked[:rerank_k] ] else: top = [ - RetrievedChunk(text=c["text"], bucket=c["bucket"], user=c["user"], score=1.0) + RetrievedChunk( + text=c["text"], bucket=c["bucket"], user=c["user"], score=1.0 + ) for c in candidates[:rerank_k] ] - t_rerank = time.perf_counter() - t0 - if debug: - return top, {"t_embed": t_embed, "t_faiss": t_faiss, "t_rerank": t_rerank} return top # ── Index builder ────────────────────────────────────────────────────────────── -def build_index(persona_path: str | Path) -> tuple[faiss.Index, list[dict]]: - """Embed all memory chunks for a persona and build a FAISS IndexFlatIP.""" + +def build_index(persona_path: str | Path): with open(persona_path) as f: persona = json.load(f) @@ -132,15 +112,16 @@ def build_index(persona_path: str | Path) -> tuple[faiss.Index, list[dict]]: vecs = embedder.encode(chunks, convert_to_numpy=True, normalize_embeddings=True) dim = vecs.shape[1] + faiss = _get_faiss() index = faiss.IndexFlatIP(dim) index.add(vecs.astype(np.float32)) return index, meta -def save_index(index: faiss.Index, meta: list[dict], save_dir: str | Path) -> None: +def save_index(index, meta: list[dict], save_dir: str | Path) -> None: p = Path(save_dir) p.mkdir(parents=True, exist_ok=True) - faiss.write_index(index, str(p / "index.faiss")) + _get_faiss().write_index(index, str(p / "index.faiss")) with open(p / "meta.json", "w") as f: json.dump(meta, f, indent=2) @@ -149,7 +130,6 @@ def build_all( memories_dir: str | Path | None = None, store_dir: str | Path | None = None, ) -> None: - """Rebuild FAISS indexes for all personas using the configured BGE embedder.""" memories_dir = Path(memories_dir or settings.memories_dir) store_dir = Path(store_dir or settings.faiss_store_dir) diff --git a/backend/sensing/__init__.py b/backend/sensing/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/sensing/air_writing.py b/backend/sensing/air_writing.py similarity index 76% rename from sensing/air_writing.py rename to backend/sensing/air_writing.py index 15913925afccfeca3aa26def645bca9f6a03be55..0135513da9fa66a57c93fb906fec60c535434c78 100644 --- a/sensing/air_writing.py +++ b/backend/sensing/air_writing.py @@ -1,35 +1,14 @@ -""" -L1 — Air writing recognition via index-finger tip trajectory (proposal §5.2). - -Tracks MediaPipe Hands landmark 8 (index fingertip) across frames. -Stroke segmentation uses velocity thresholding: - - stroke starts when velocity > START_VEL px/frame - - stroke ends when velocity < END_VEL px/frame for > GAP_MS ms - -Segmented strokes are classified against a template library using -Dynamic Time Warping (DTW). Supports: - - 26 uppercase English letters (A-Z) - - 10 digits (0-9) - - 10 most frequent Devanagari characters (for Arjun's Hindi inputs) - -Recognised characters are concatenated and returned as a text string -to the intent decomposition layer. -""" +# Air writing recognition — fingertip trajectory → DTW character matching. from __future__ import annotations import time -from collections import deque from dataclasses import dataclass, field import numpy as np -from config.settings import settings +from backend.config.settings import settings -try: - import mediapipe as mp - _MP_AVAILABLE = True -except ImportError: - _MP_AVAILABLE = False +mp = None # ── Landmark index ───────────────────────────────────────────────────────────── _INDEX_TIP = 8 @@ -41,6 +20,7 @@ class AirWriter: Stateful air-writing recogniser. Feed frames from a webcam loop. Call `get_text()` to retrieve and clear the current buffer. """ + _trajectory: list[tuple[float, float]] = field(default_factory=list) _in_stroke: bool = False _stroke_end_time: float = field(default=0.0) @@ -48,8 +28,9 @@ class AirWriter: _templates: dict[str, np.ndarray] = field(default_factory=dict) def __post_init__(self): - if not _MP_AVAILABLE: - raise ImportError("mediapipe is required: pip install mediapipe") + global mp + import mediapipe as mp + self._hands = mp.solutions.hands.Hands( static_image_mode=False, max_num_hands=1, @@ -65,6 +46,7 @@ class AirWriter: completes, or None otherwise. """ import cv2 + rgb = cv2.cvtColor(bgr_frame, cv2.COLOR_BGR2RGB) result = self._hands.process(rgb) @@ -82,7 +64,7 @@ class AirWriter: self._prev_pt = tip start_v = settings.air_write_velocity_start - end_v = settings.air_write_velocity_end + end_v = settings.air_write_velocity_end if velocity > start_v: self._in_stroke = True @@ -133,6 +115,7 @@ class AirWriter: # ── DTW helpers ─────────────────────────────────────────────────────────────── + def _normalise_trajectory(pts: np.ndarray) -> np.ndarray: """Scale trajectory to unit bounding box, resample to 32 points.""" pts = pts - pts.min(axis=0) @@ -141,10 +124,12 @@ def _normalise_trajectory(pts: np.ndarray) -> np.ndarray: # Resample to fixed length via linear interpolation t_old = np.linspace(0, 1, len(pts)) t_new = np.linspace(0, 1, 32) - return np.column_stack([ - np.interp(t_new, t_old, pts[:, 0]), - np.interp(t_new, t_old, pts[:, 1]), - ]) + return np.column_stack( + [ + np.interp(t_new, t_old, pts[:, 0]), + np.interp(t_new, t_old, pts[:, 1]), + ] + ) def _dtw_distance(a: np.ndarray, b: np.ndarray) -> float: @@ -160,17 +145,11 @@ def _dtw_distance(a: np.ndarray, b: np.ndarray) -> float: def _load_templates() -> dict[str, np.ndarray]: - """ - Load pre-recorded stroke templates from disk. - Template files should be numpy arrays of shape (32, 2) stored as .npy. - Returns an empty dict if no template directory exists yet. - """ - from pathlib import Path - template_dir = Path("data/air_write_templates") + template_dir = settings.data_dir / "air_write_templates" if not template_dir.exists(): return {} templates = {} for f in template_dir.glob("*.npy"): - char = f.stem # filename = character label + char = f.stem # filename = character label templates[char] = np.load(f) return templates diff --git a/sensing/face_mesh.py b/backend/sensing/face_mesh.py similarity index 65% rename from sensing/face_mesh.py rename to backend/sensing/face_mesh.py index b10029e3f03b07cbf25ff0bd9e1a68cc40097bcc..fc75b10d9c96497240901697865af4c13e126324 100644 --- a/sensing/face_mesh.py +++ b/backend/sensing/face_mesh.py @@ -1,61 +1,37 @@ -""" -L1 — Facial affect detection via MediaPipe 2D Face Mesh. - -Extracts 4 geometric features from 478 landmarks at ~10 fps: - MAR — Mouth Aspect Ratio (surprise / speech attempt) - EAR — Eye Aspect Ratio (frustration / blink) - BRI — Brow Raise Index (surprise / questioning) - LCP — Lip Corner Pull (smile vs frown) - -These form the affect vector fed into MobileNetV3-Small affect classifier, -which maps to one of 4 actionable states: HAPPY | FRUSTRATED | NEUTRAL | SURPRISED. - -EMA smoothing (α=0.3) prevents transient expressions (sneezes, blinks) -from destabilising the detected state across turns. -""" +# Facial affect detection via MediaPipe Face Mesh (MAR/EAR/BRI/LCP → emotion). from __future__ import annotations -import time from dataclasses import dataclass, field import numpy as np -from config.settings import settings -from pipeline.state import AffectState, AffectVector - -try: - import mediapipe as mp - _MP_AVAILABLE = True -except ImportError: - _MP_AVAILABLE = False +from backend.config.settings import settings +from backend.pipeline.state import AffectState, AffectVector -try: - import cv2 - _CV2_AVAILABLE = True -except ImportError: - _CV2_AVAILABLE = False +mp = None +cv2 = None -# ── MediaPipe landmark indices (from proposal §5.2) ─────────────────────────── +# ── MediaPipe landmark indices ──────────────────────────────────────────────── # MAR — mouth vertical / horizontal ratio -_MOUTH_TOP = 13 +_MOUTH_TOP = 13 _MOUTH_BOTTOM = 14 -_MOUTH_LEFT = 61 -_MOUTH_RIGHT = 291 +_MOUTH_LEFT = 61 +_MOUTH_RIGHT = 291 # EAR — eye vertical / horizontal ratio (right eye) -_EYE_TOP = 159 +_EYE_TOP = 159 _EYE_BOTTOM = 145 -_EYE_LEFT = 33 -_EYE_RIGHT = 133 +_EYE_LEFT = 33 +_EYE_RIGHT = 133 # BRI — brow vertical displacement relative to eye centre -_BROW_LEFT = 70 +_BROW_LEFT = 70 _BROW_RIGHT = 300 # LCP — mouth corner horizontal displacement from neutral baseline -_CORNER_LEFT = 61 +_CORNER_LEFT = 61 _CORNER_RIGHT = 291 @@ -70,20 +46,22 @@ class AffectDetector: Stateful detector that maintains EMA-smoothed affect across frames. Create one instance per session and call `process_frame` each frame. """ - _smoothed: AffectVector = field(default_factory=lambda: AffectVector(MAR=0.0, EAR=0.3, BRI=0.0, LCP=0.0)) - _neutral_lcp: float = 0.0 # calibrated at session start + + _smoothed: AffectVector = field( + default_factory=lambda: AffectVector(MAR=0.0, EAR=0.3, BRI=0.0, LCP=0.0) + ) + _neutral_lcp: float = 0.0 # calibrated at session start _calibrated: bool = False def __post_init__(self): - if not _MP_AVAILABLE: - raise ImportError("mediapipe is required: pip install mediapipe") - if not _CV2_AVAILABLE: - raise ImportError("opencv-python is required: pip install opencv-python") + global mp, cv2 + import cv2 + import mediapipe as mp self._face_mesh = mp.solutions.face_mesh.FaceMesh( static_image_mode=False, max_num_faces=1, - refine_landmarks=True, # enables iris landmarks (468-477) + refine_landmarks=True, # enables iris landmarks (468-477) min_detection_confidence=0.5, min_tracking_confidence=0.5, ) @@ -146,14 +124,15 @@ class AffectDetector: # LCP — average horizontal mouth corner displacement LCP = float((pt(_CORNER_LEFT)[0] + pt(_CORNER_RIGHT)[0]) / 2) - return {"MAR": float(MAR), "EAR": float(EAR), "BRI": float(BRI), "LCP": float(LCP)} + return { + "MAR": float(MAR), + "EAR": float(EAR), + "BRI": float(BRI), + "LCP": float(LCP), + } @staticmethod def _classify(v: AffectVector) -> str: - """ - Rule-based classifier over the 4 geometric features. - Replace with MobileNetV3-Small for final evaluation. - """ if v["BRI"] > 0.25 and v["MAR"] > 0.3: return "SURPRISED" if v["EAR"] < 0.15 and v["LCP"] < -5: diff --git a/sensing/gaze.py b/backend/sensing/gaze.py similarity index 71% rename from sensing/gaze.py rename to backend/sensing/gaze.py index 52039d9c5ac1c553a5c65b11ebf2219d7d1c2053..3679de38f8cd43ebe1d0f6ddc0c2112f36571d79 100644 --- a/sensing/gaze.py +++ b/backend/sensing/gaze.py @@ -1,47 +1,27 @@ -""" -L1 — Gaze-based retrieval activation (Bonus feature, proposal §5.2). - -Uses MediaPipe iris landmarks (468-472) to estimate gaze direction as -a 2D screen-coordinate vector. Sustained fixation (> 1.5 s dwell time) -on a defined UI region pre-biases the retrieval layer toward the -corresponding memory bucket. - -UI region → bucket mapping: - top-left quadrant → family - top-right quadrant → medical - bottom-left quadrant → hobbies - bottom-right quadrant → daily_routine - centre strip → social -""" +# Gaze-based retrieval bucket hinting via MediaPipe iris landmarks. from __future__ import annotations import time from dataclasses import dataclass, field -import numpy as np +from backend.config.settings import settings -from config.settings import settings - -try: - import mediapipe as mp - _MP_AVAILABLE = True -except ImportError: - _MP_AVAILABLE = False +mp = None # ── Iris landmark indices ────────────────────────────────────────────────────── # MediaPipe refine_landmarks=True adds iris landmarks 468-477 -_LEFT_IRIS_CENTER = 468 +_LEFT_IRIS_CENTER = 468 _RIGHT_IRIS_CENTER = 473 # ── Screen region → bucket map ───────────────────────────────────────────────── # Defined as (x_min, y_min, x_max, y_max) in normalised [0,1] coords _REGION_BUCKET: list[tuple[tuple[float, float, float, float], str]] = [ + ((0.3, 0.3, 0.7, 0.7), "social"), # centre checked first (most specific) ((0.0, 0.0, 0.5, 0.5), "family"), ((0.5, 0.0, 1.0, 0.5), "medical"), ((0.0, 0.5, 0.5, 1.0), "hobbies"), ((0.5, 0.5, 1.0, 1.0), "daily_routine"), - ((0.3, 0.3, 0.7, 0.7), "social"), # centre strip (checked last → lowest priority) ] @@ -51,12 +31,14 @@ class GazeTracker: Stateful gaze tracker. Call `process_frame` each frame. Returns the bucket name when dwell threshold is exceeded, else None. """ + _dwell_start: float = field(default=0.0) _current_region: str | None = field(default=None) def __post_init__(self): - if not _MP_AVAILABLE: - raise ImportError("mediapipe is required: pip install mediapipe") + global mp + import mediapipe as mp + self._face_mesh = mp.solutions.face_mesh.FaceMesh( static_image_mode=False, max_num_faces=1, @@ -66,11 +48,8 @@ class GazeTracker: ) def process_frame(self, bgr_frame) -> str | None: - """ - Returns the hinted bucket name once dwell threshold is exceeded, - then resets the dwell timer. Returns None otherwise. - """ import cv2 + rgb = cv2.cvtColor(bgr_frame, cv2.COLOR_BGR2RGB) result = self._face_mesh.process(rgb) diff --git a/sensing/gesture.py b/backend/sensing/gesture.py similarity index 63% rename from sensing/gesture.py rename to backend/sensing/gesture.py index bbd08c4a1b599b388b635c35483ee1e68f96a09a..e6e3321ddca3d5104fac186d6dcbbdded2e88639 100644 --- a/sensing/gesture.py +++ b/backend/sensing/gesture.py @@ -1,34 +1,17 @@ -""" -L1 — Hand gesture recognition via MediaPipe Hands. - -Recognises 4 gestures from 21 3D hand landmarks at ~15 fps using -normalised joint-angle rules (no ML model needed at this stage): - - THUMBS_UP → [TONE:AFFIRMATIVE] - THUMBS_DOWN → [TONE:NEGATIVE] - POINTING → [INTENT:REFERENTIAL] - WAVING → [INTENT:GREETING] - -Each detected gesture is mapped to a stylistic constraint tag that is -injected into the generation prompt by the planner node. -""" +# Hand gesture recognition via MediaPipe Hands. from __future__ import annotations import numpy as np -try: - import mediapipe as mp - _MP_AVAILABLE = True -except ImportError: - _MP_AVAILABLE = False +mp = None # Gesture → prompt constraint tag mapping GESTURE_TO_TAG: dict[str, str] = { - "THUMBS_UP": "[GESTURE:THUMBS_UP][TONE:AFFIRMATIVE]", + "THUMBS_UP": "[GESTURE:THUMBS_UP][TONE:AFFIRMATIVE]", "THUMBS_DOWN": "[GESTURE:THUMBS_DOWN][TONE:NEGATIVE]", - "POINTING": "[GESTURE:POINTING][INTENT:REFERENTIAL]", - "WAVING": "[GESTURE:WAVING][INTENT:GREETING]", + "POINTING": "[GESTURE:POINTING][INTENT:REFERENTIAL]", + "WAVING": "[GESTURE:WAVING][INTENT:GREETING]", } @@ -39,8 +22,9 @@ class GestureClassifier: """ def __init__(self): - if not _MP_AVAILABLE: - raise ImportError("mediapipe is required: pip install mediapipe") + global mp + import mediapipe as mp + self._hands = mp.solutions.hands.Hands( static_image_mode=False, max_num_hands=1, @@ -53,6 +37,7 @@ class GestureClassifier: Returns a gesture label string or None if no clear gesture is detected. """ import cv2 + rgb = cv2.cvtColor(bgr_frame, cv2.COLOR_BGR2RGB) result = self._hands.process(rgb) @@ -71,27 +56,21 @@ class GestureClassifier: @staticmethod def _classify(pts: np.ndarray) -> str | None: - """ - Rule-based gesture classification over normalised joint positions. - - MediaPipe hand landmark indices: - 0=WRIST, 1-4=THUMB, 5-8=INDEX, 9-12=MIDDLE, 13-16=RING, 17-20=PINKY - """ # Normalise: wrist at origin, scale by palm width wrist = pts[0] palm_width = np.linalg.norm(pts[5] - pts[17]) + 1e-6 p = (pts - wrist) / palm_width - thumb_tip = p[4] - index_tip = p[8] - middle_tip = p[12] - ring_tip = p[16] - pinky_tip = p[20] - index_mcp = p[5] # knuckle + thumb_tip = p[4] + index_tip = p[8] + middle_tip = p[12] + ring_tip = p[16] + pinky_tip = p[20] + index_mcp = p[5] # knuckle # THUMBS_UP: thumb tip above wrist, other fingers curled fingers_curled = all( - np.linalg.norm(tip) < np.linalg.norm(p[mcp]) + np.linalg.norm(tip) < np.linalg.norm(mcp) for tip, mcp in [(index_tip, p[5]), (middle_tip, p[9]), (ring_tip, p[13])] ) if thumb_tip[1] < -0.3 and fingers_curled: @@ -103,9 +82,8 @@ class GestureClassifier: # POINTING: index extended, others curled index_extended = np.linalg.norm(index_tip) > np.linalg.norm(index_mcp) * 1.3 - others_curled = all( - np.linalg.norm(tip) < 0.5 - for tip in [middle_tip, ring_tip, pinky_tip] + others_curled = all( + np.linalg.norm(tip) < 0.5 for tip in [middle_tip, ring_tip, pinky_tip] ) if index_extended and others_curled: return "POINTING" diff --git a/ui/app.py b/backend/ui/app.py similarity index 85% rename from ui/app.py rename to backend/ui/app.py index 0b7592ec4220c3c3cff539c86d24c6b50737cbe0..3f59cb6da2f3f5bb3c2a7ddf8f86e029dec3fed3 100644 --- a/ui/app.py +++ b/backend/ui/app.py @@ -8,10 +8,8 @@ Panels: Run: streamlit run ui/app.py """ -from __future__ import annotations -import json -import time +from __future__ import annotations import requests import streamlit as st @@ -52,8 +50,11 @@ with st.sidebar: st.error("API not reachable — start the FastAPI server first.") user_options = {u["id"]: f"{u['name']} ({u['condition']})" for u in users} - selected = st.selectbox("Select persona", options=list(user_options.keys()), - format_func=lambda k: user_options.get(k, k)) + selected = st.selectbox( + "Select persona", + options=list(user_options.keys()), + format_func=lambda k: user_options.get(k, k), + ) if selected != st.session_state.user_id: st.session_state.user_id = selected @@ -73,15 +74,19 @@ with st.sidebar: ["Auto (webcam)", "HAPPY", "FRUSTRATED", "NEUTRAL", "SURPRISED"], index=0, ) - st.session_state.affect_override = None if affect_choice == "Auto (webcam)" else affect_choice + st.session_state.affect_override = ( + None if affect_choice == "Auto (webcam)" else affect_choice + ) st.divider() # Live affect indicator st.subheader("Detected Affect") affect_emoji = { - "HAPPY": "😊", "FRUSTRATED": "😤", - "NEUTRAL": "😐", "SURPRISED": "😲", + "HAPPY": "😊", + "FRUSTRATED": "😤", + "NEUTRAL": "😐", + "SURPRISED": "😲", } af = st.session_state.last_affect st.markdown(f"### {affect_emoji.get(af, '❓')} {af}") @@ -89,7 +94,9 @@ with st.sidebar: # Webcam placeholder st.divider() st.subheader("Webcam Feed") - st.info("Live webcam sensing runs in the sensing client.\nAffect is sent to the API automatically.") + st.info( + "Live webcam sensing runs in the sensing client.\nAffect is sent to the API automatically." + ) # ── Main chat area ───────────────────────────────────────────────────────────── @@ -119,12 +126,15 @@ with chat_col: "affect_override": st.session_state.affect_override, } resp = requests.post(f"{API_BASE}/chat", json=payload, timeout=15) + resp.raise_for_status() data = resp.json() response_text = data.get("response", "I don't know.") st.markdown(f"**AAC User:** {response_text}") - st.session_state.messages.append({"role": "aac_user", "content": response_text}) + st.session_state.messages.append( + {"role": "aac_user", "content": response_text} + ) st.session_state.last_affect = data.get("affect", "NEUTRAL") st.session_state.last_latency = data.get("latency", {}) @@ -141,11 +151,11 @@ with metrics_col: lat = st.session_state.last_latency if lat: for key, label in [ - ("t_sensing", "Sensing"), - ("t_intent", "Intent"), - ("t_retrieval", "Retrieval"), + ("t_sensing", "Sensing"), + ("t_intent", "Intent"), + ("t_retrieval", "Retrieval"), ("t_generation", "Generation"), - ("t_total", "**Total**"), + ("t_total", "**Total**"), ]: val = lat.get(key, 0.0) st.metric(label=label, value=f"{val:.3f}s") diff --git a/config/__init__.py b/config/__init__.py deleted file mode 100644 index 2d422317a908f97c8613546a35a175f0b20cd1bc..0000000000000000000000000000000000000000 --- a/config/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from config.settings import settings - -__all__ = ["settings"] diff --git a/frontend/.gitignore b/frontend/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..a547bf36d8d11a4f89c59c144f24795749086dd1 --- /dev/null +++ b/frontend/.gitignore @@ -0,0 +1,24 @@ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* +lerna-debug.log* + +node_modules +dist +dist-ssr +*.local + +# Editor directories and files +.vscode/* +!.vscode/extensions.json +.idea +.DS_Store +*.suo +*.ntvs* +*.njsproj +*.sln +*.sw? diff --git a/frontend/README.md b/frontend/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7dbf7ebf3b2a3d84ad526bc47810d1d211331b8b --- /dev/null +++ b/frontend/README.md @@ -0,0 +1,73 @@ +# React + TypeScript + Vite + +This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules. + +Currently, two official plugins are available: + +- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react) uses [Oxc](https://oxc.rs) +- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react-swc) uses [SWC](https://swc.rs/) + +## React Compiler + +The React Compiler is not enabled on this template because of its impact on dev & build performances. To add it, see [this documentation](https://react.dev/learn/react-compiler/installation). + +## Expanding the ESLint configuration + +If you are developing a production application, we recommend updating the configuration to enable type-aware lint rules: + +```js +export default defineConfig([ + globalIgnores(['dist']), + { + files: ['**/*.{ts,tsx}'], + extends: [ + // Other configs... + + // Remove tseslint.configs.recommended and replace with this + tseslint.configs.recommendedTypeChecked, + // Alternatively, use this for stricter rules + tseslint.configs.strictTypeChecked, + // Optionally, add this for stylistic rules + tseslint.configs.stylisticTypeChecked, + + // Other configs... + ], + languageOptions: { + parserOptions: { + project: ['./tsconfig.node.json', './tsconfig.app.json'], + tsconfigRootDir: import.meta.dirname, + }, + // other options... + }, + }, +]) +``` + +You can also install [eslint-plugin-react-x](https://github.com/Rel1cx/eslint-react/tree/main/packages/plugins/eslint-plugin-react-x) and [eslint-plugin-react-dom](https://github.com/Rel1cx/eslint-react/tree/main/packages/plugins/eslint-plugin-react-dom) for React-specific lint rules: + +```js +// eslint.config.js +import reactX from 'eslint-plugin-react-x' +import reactDom from 'eslint-plugin-react-dom' + +export default defineConfig([ + globalIgnores(['dist']), + { + files: ['**/*.{ts,tsx}'], + extends: [ + // Other configs... + // Enable lint rules for React + reactX.configs['recommended-typescript'], + // Enable lint rules for React DOM + reactDom.configs.recommended, + ], + languageOptions: { + parserOptions: { + project: ['./tsconfig.node.json', './tsconfig.app.json'], + tsconfigRootDir: import.meta.dirname, + }, + // other options... + }, + }, +]) +``` diff --git a/frontend/eslint.config.js b/frontend/eslint.config.js new file mode 100644 index 0000000000000000000000000000000000000000..5e6b472f583e34a1cca751440d4f241495475723 --- /dev/null +++ b/frontend/eslint.config.js @@ -0,0 +1,23 @@ +import js from '@eslint/js' +import globals from 'globals' +import reactHooks from 'eslint-plugin-react-hooks' +import reactRefresh from 'eslint-plugin-react-refresh' +import tseslint from 'typescript-eslint' +import { defineConfig, globalIgnores } from 'eslint/config' + +export default defineConfig([ + globalIgnores(['dist']), + { + files: ['**/*.{ts,tsx}'], + extends: [ + js.configs.recommended, + tseslint.configs.recommended, + reactHooks.configs.flat.recommended, + reactRefresh.configs.vite, + ], + languageOptions: { + ecmaVersion: 2020, + globals: globals.browser, + }, + }, +]) diff --git a/frontend/index.html b/frontend/index.html new file mode 100644 index 0000000000000000000000000000000000000000..0fca6f0434183b2c6f46d5336c59f6954c56b470 --- /dev/null +++ b/frontend/index.html @@ -0,0 +1,13 @@ + + + + + + + frontend + + +
+ + + diff --git a/frontend/package.json b/frontend/package.json new file mode 100644 index 0000000000000000000000000000000000000000..9fd0a1f388905cbeda37448c8db8756950161594 --- /dev/null +++ b/frontend/package.json @@ -0,0 +1,31 @@ +{ + "name": "frontend", + "private": true, + "version": "0.0.0", + "type": "module", + "scripts": { + "dev": "vite", + "build": "tsc -b && vite build", + "lint": "eslint .", + "preview": "vite preview" + }, + "dependencies": { + "@mediapipe/tasks-vision": "^0.10.34", + "react": "^19.2.4", + "react-dom": "^19.2.4" + }, + "devDependencies": { + "@eslint/js": "^9.39.4", + "@types/node": "^24.12.2", + "@types/react": "^19.2.14", + "@types/react-dom": "^19.2.3", + "@vitejs/plugin-react": "^6.0.1", + "eslint": "^9.39.4", + "eslint-plugin-react-hooks": "^7.0.1", + "eslint-plugin-react-refresh": "^0.5.2", + "globals": "^17.4.0", + "typescript": "~6.0.2", + "typescript-eslint": "^8.58.0", + "vite": "^8.0.8" + } +} diff --git a/frontend/pnpm-lock.yaml b/frontend/pnpm-lock.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ba28d3f3e20cb1cedf27dbad8ddb345df99dff26 --- /dev/null +++ b/frontend/pnpm-lock.yaml @@ -0,0 +1,1863 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + '@mediapipe/tasks-vision': + specifier: ^0.10.34 + version: 0.10.34 + react: + specifier: ^19.2.4 + version: 19.2.5 + react-dom: + specifier: ^19.2.4 + version: 19.2.5(react@19.2.5) + devDependencies: + '@eslint/js': + specifier: ^9.39.4 + version: 9.39.4 + '@types/node': + specifier: ^24.12.2 + version: 24.12.2 + '@types/react': + specifier: ^19.2.14 + version: 19.2.14 + '@types/react-dom': + specifier: ^19.2.3 + version: 19.2.3(@types/react@19.2.14) + '@vitejs/plugin-react': + specifier: ^6.0.1 + version: 6.0.1(vite@8.0.8(@types/node@24.12.2)) + eslint: + specifier: ^9.39.4 + version: 9.39.4 + eslint-plugin-react-hooks: + specifier: ^7.0.1 + version: 7.0.1(eslint@9.39.4) + eslint-plugin-react-refresh: + specifier: ^0.5.2 + version: 0.5.2(eslint@9.39.4) + globals: + specifier: ^17.4.0 + version: 17.5.0 + typescript: + specifier: ~6.0.2 + version: 6.0.2 + typescript-eslint: + specifier: ^8.58.0 + version: 8.58.2(eslint@9.39.4)(typescript@6.0.2) + vite: + specifier: ^8.0.8 + version: 8.0.8(@types/node@24.12.2) + +packages: + + '@babel/code-frame@7.29.0': + resolution: {integrity: sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw==} + engines: {node: '>=6.9.0'} + + '@babel/compat-data@7.29.0': + resolution: {integrity: sha512-T1NCJqT/j9+cn8fvkt7jtwbLBfLC/1y1c7NtCeXFRgzGTsafi68MRv8yzkYSapBnFA6L3U2VSc02ciDzoAJhJg==} + engines: {node: '>=6.9.0'} + + '@babel/core@7.29.0': + resolution: {integrity: sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==} + engines: {node: '>=6.9.0'} + + '@babel/generator@7.29.1': + resolution: {integrity: sha512-qsaF+9Qcm2Qv8SRIMMscAvG4O3lJ0F1GuMo5HR/Bp02LopNgnZBC/EkbevHFeGs4ls/oPz9v+Bsmzbkbe+0dUw==} + engines: {node: '>=6.9.0'} + + '@babel/helper-compilation-targets@7.28.6': + resolution: {integrity: sha512-JYtls3hqi15fcx5GaSNL7SCTJ2MNmjrkHXg4FSpOA/grxK8KwyZ5bubHsCq8FXCkua6xhuaaBit+3b7+VZRfcA==} + engines: {node: '>=6.9.0'} + + '@babel/helper-globals@7.28.0': + resolution: {integrity: sha512-+W6cISkXFa1jXsDEdYA8HeevQT/FULhxzR99pxphltZcVaugps53THCeiWA8SguxxpSp3gKPiuYfSWopkLQ4hw==} + engines: {node: '>=6.9.0'} + + '@babel/helper-module-imports@7.28.6': + resolution: {integrity: sha512-l5XkZK7r7wa9LucGw9LwZyyCUscb4x37JWTPz7swwFE/0FMQAGpiWUZn8u9DzkSBWEcK25jmvubfpw2dnAMdbw==} + engines: {node: '>=6.9.0'} + + '@babel/helper-module-transforms@7.28.6': + resolution: {integrity: sha512-67oXFAYr2cDLDVGLXTEABjdBJZ6drElUSI7WKp70NrpyISso3plG9SAGEF6y7zbha/wOzUByWWTJvEDVNIUGcA==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 + + '@babel/helper-string-parser@7.27.1': + resolution: {integrity: sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==} + engines: {node: '>=6.9.0'} + + '@babel/helper-validator-identifier@7.28.5': + resolution: {integrity: sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==} + engines: {node: '>=6.9.0'} + + '@babel/helper-validator-option@7.27.1': + resolution: {integrity: sha512-YvjJow9FxbhFFKDSuFnVCe2WxXk1zWc22fFePVNEaWJEu8IrZVlda6N0uHwzZrUM1il7NC9Mlp4MaJYbYd9JSg==} + engines: {node: '>=6.9.0'} + + '@babel/helpers@7.29.2': + resolution: {integrity: sha512-HoGuUs4sCZNezVEKdVcwqmZN8GoHirLUcLaYVNBK2J0DadGtdcqgr3BCbvH8+XUo4NGjNl3VOtSjEKNzqfFgKw==} + engines: {node: '>=6.9.0'} + + '@babel/parser@7.29.2': + resolution: {integrity: sha512-4GgRzy/+fsBa72/RZVJmGKPmZu9Byn8o4MoLpmNe1m8ZfYnz5emHLQz3U4gLud6Zwl0RZIcgiLD7Uq7ySFuDLA==} + engines: {node: '>=6.0.0'} + hasBin: true + + '@babel/template@7.28.6': + resolution: {integrity: sha512-YA6Ma2KsCdGb+WC6UpBVFJGXL58MDA6oyONbjyF/+5sBgxY/dwkhLogbMT2GXXyU84/IhRw/2D1Os1B/giz+BQ==} + engines: {node: '>=6.9.0'} + + '@babel/traverse@7.29.0': + resolution: {integrity: sha512-4HPiQr0X7+waHfyXPZpWPfWL/J7dcN1mx9gL6WdQVMbPnF3+ZhSMs8tCxN7oHddJE9fhNE7+lxdnlyemKfJRuA==} + engines: {node: '>=6.9.0'} + + '@babel/types@7.29.0': + resolution: {integrity: sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A==} + engines: {node: '>=6.9.0'} + + '@emnapi/core@1.9.2': + resolution: {integrity: sha512-UC+ZhH3XtczQYfOlu3lNEkdW/p4dsJ1r/bP7H8+rhao3TTTMO1ATq/4DdIi23XuGoFY+Cz0JmCbdVl0hz9jZcA==} + + '@emnapi/runtime@1.9.2': + resolution: {integrity: sha512-3U4+MIWHImeyu1wnmVygh5WlgfYDtyf0k8AbLhMFxOipihf6nrWC4syIm/SwEeec0mNSafiiNnMJwbza/Is6Lw==} + + '@emnapi/wasi-threads@1.2.1': + resolution: {integrity: sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==} + + '@eslint-community/eslint-utils@4.9.1': + resolution: {integrity: sha512-phrYmNiYppR7znFEdqgfWHXR6NCkZEK7hwWDHZUjit/2/U0r6XvkDl0SYnoM51Hq7FhCGdLDT6zxCCOY1hexsQ==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + peerDependencies: + eslint: ^6.0.0 || ^7.0.0 || >=8.0.0 + + '@eslint-community/regexpp@4.12.2': + resolution: {integrity: sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew==} + engines: {node: ^12.0.0 || ^14.0.0 || >=16.0.0} + + '@eslint/config-array@0.21.2': + resolution: {integrity: sha512-nJl2KGTlrf9GjLimgIru+V/mzgSK0ABCDQRvxw5BjURL7WfH5uoWmizbH7QB6MmnMBd8cIC9uceWnezL1VZWWw==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@eslint/config-helpers@0.4.2': + resolution: {integrity: sha512-gBrxN88gOIf3R7ja5K9slwNayVcZgK6SOUORm2uBzTeIEfeVaIhOpCtTox3P6R7o2jLFwLFTLnC7kU/RGcYEgw==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@eslint/core@0.17.0': + resolution: {integrity: sha512-yL/sLrpmtDaFEiUj1osRP4TI2MDz1AddJL+jZ7KSqvBuliN4xqYY54IfdN8qD8Toa6g1iloph1fxQNkjOxrrpQ==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@eslint/eslintrc@3.3.5': + resolution: {integrity: sha512-4IlJx0X0qftVsN5E+/vGujTRIFtwuLbNsVUe7TO6zYPDR1O6nFwvwhIKEKSrl6dZchmYBITazxKoUYOjdtjlRg==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@eslint/js@9.39.4': + resolution: {integrity: sha512-nE7DEIchvtiFTwBw4Lfbu59PG+kCofhjsKaCWzxTpt4lfRjRMqG6uMBzKXuEcyXhOHoUp9riAm7/aWYGhXZ9cw==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@eslint/object-schema@2.1.7': + resolution: {integrity: sha512-VtAOaymWVfZcmZbp6E2mympDIHvyjXs/12LqWYjVw6qjrfF+VK+fyG33kChz3nnK+SU5/NeHOqrTEHS8sXO3OA==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@eslint/plugin-kit@0.4.1': + resolution: {integrity: sha512-43/qtrDUokr7LJqoF2c3+RInu/t4zfrpYdoSDfYyhg52rwLV6TnOvdG4fXm7IkSB3wErkcmJS9iEhjVtOSEjjA==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@humanfs/core@0.19.1': + resolution: {integrity: sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==} + engines: {node: '>=18.18.0'} + + '@humanfs/node@0.16.7': + resolution: {integrity: sha512-/zUx+yOsIrG4Y43Eh2peDeKCxlRt/gET6aHfaKpuq267qXdYDFViVHfMaLyygZOnl0kGWxFIgsBy8QFuTLUXEQ==} + engines: {node: '>=18.18.0'} + + '@humanwhocodes/module-importer@1.0.1': + resolution: {integrity: sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==} + engines: {node: '>=12.22'} + + '@humanwhocodes/retry@0.4.3': + resolution: {integrity: sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ==} + engines: {node: '>=18.18'} + + '@jridgewell/gen-mapping@0.3.13': + resolution: {integrity: sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==} + + '@jridgewell/remapping@2.3.5': + resolution: {integrity: sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==} + + '@jridgewell/resolve-uri@3.1.2': + resolution: {integrity: sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==} + engines: {node: '>=6.0.0'} + + '@jridgewell/sourcemap-codec@1.5.5': + resolution: {integrity: sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==} + + '@jridgewell/trace-mapping@0.3.31': + resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} + + '@mediapipe/tasks-vision@0.10.34': + resolution: {integrity: sha512-KFGyhDsjJ+9WUMcMfjTOpcEp3LJNS3KwC7BfvKrCYELn/7G/5kmwnU7z6Spps+iWQoTGL8xW8i68r65OTa3DwA==} + + '@napi-rs/wasm-runtime@1.1.4': + resolution: {integrity: sha512-3NQNNgA1YSlJb/kMH1ildASP9HW7/7kYnRI2szWJaofaS1hWmbGI4H+d3+22aGzXXN9IJ+n+GiFVcGipJP18ow==} + peerDependencies: + '@emnapi/core': ^1.7.1 + '@emnapi/runtime': ^1.7.1 + + '@oxc-project/types@0.124.0': + resolution: {integrity: sha512-VBFWMTBvHxS11Z5Lvlr3IWgrwhMTXV+Md+EQF0Xf60+wAdsGFTBx7X7K/hP4pi8N7dcm1RvcHwDxZ16Qx8keUg==} + + '@rolldown/binding-android-arm64@1.0.0-rc.15': + resolution: {integrity: sha512-YYe6aWruPZDtHNpwu7+qAHEMbQ/yRl6atqb/AhznLTnD3UY99Q1jE7ihLSahNWkF4EqRPVC4SiR4O0UkLK02tA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [android] + + '@rolldown/binding-darwin-arm64@1.0.0-rc.15': + resolution: {integrity: sha512-oArR/ig8wNTPYsXL+Mzhs0oxhxfuHRfG7Ikw7jXsw8mYOtk71W0OkF2VEVh699pdmzjPQsTjlD1JIOoHkLP1Fg==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [darwin] + + '@rolldown/binding-darwin-x64@1.0.0-rc.15': + resolution: {integrity: sha512-YzeVqOqjPYvUbJSWJ4EDL8ahbmsIXQpgL3JVipmN+MX0XnXMeWomLN3Fb+nwCmP/jfyqte5I3XRSm7OfQrbyxw==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [darwin] + + '@rolldown/binding-freebsd-x64@1.0.0-rc.15': + resolution: {integrity: sha512-9Erhx956jeQ0nNTyif1+QWAXDRD38ZNjr//bSHrt6wDwB+QkAfl2q6Mn1k6OBPerznjRmbM10lgRb1Pli4xZPw==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [freebsd] + + '@rolldown/binding-linux-arm-gnueabihf@1.0.0-rc.15': + resolution: {integrity: sha512-cVwk0w8QbZJGTnP/AHQBs5yNwmpgGYStL88t4UIaqcvYJWBfS0s3oqVLZPwsPU6M0zlW4GqjP0Zq5MnAGwFeGA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm] + os: [linux] + + '@rolldown/binding-linux-arm64-gnu@1.0.0-rc.15': + resolution: {integrity: sha512-eBZ/u8iAK9SoHGanqe/jrPnY0JvBN6iXbVOsbO38mbz+ZJsaobExAm1Iu+rxa4S1l2FjG0qEZn4Rc6X8n+9M+w==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@rolldown/binding-linux-arm64-musl@1.0.0-rc.15': + resolution: {integrity: sha512-ZvRYMGrAklV9PEkgt4LQM6MjQX2P58HPAuecwYObY2DhS2t35R0I810bKi0wmaYORt6m/2Sm+Z+nFgb0WhXNcQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@rolldown/binding-linux-ppc64-gnu@1.0.0-rc.15': + resolution: {integrity: sha512-VDpgGBzgfg5hLg+uBpCLoFG5kVvEyafmfxGUV0UHLcL5irxAK7PKNeC2MwClgk6ZAiNhmo9FLhRYgvMmedLtnQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@rolldown/binding-linux-s390x-gnu@1.0.0-rc.15': + resolution: {integrity: sha512-y1uXY3qQWCzcPgRJATPSOUP4tCemh4uBdY7e3EZbVwCJTY3gLJWnQABgeUetvED+bt1FQ01OeZwvhLS2bpNrAQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@rolldown/binding-linux-x64-gnu@1.0.0-rc.15': + resolution: {integrity: sha512-023bTPBod7J3Y/4fzAN6QtpkSABR0rigtrwaP+qSEabUh5zf6ELr9Nc7GujaROuPY3uwdSIXWrvhn1KxOvurWA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@rolldown/binding-linux-x64-musl@1.0.0-rc.15': + resolution: {integrity: sha512-witB2O0/hU4CgfOOKUoeFgQ4GktPi1eEbAhaLAIpgD6+ZnhcPkUtPsoKKHRzmOoWPZue46IThdSgdo4XneOLYw==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [linux] + libc: [musl] + + '@rolldown/binding-openharmony-arm64@1.0.0-rc.15': + resolution: {integrity: sha512-UCL68NJ0Ud5zRipXZE9dF5PmirzJE4E4BCIOOssEnM7wLDsxjc6Qb0sGDxTNRTP53I6MZpygyCpY8Aa8sPfKPg==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [openharmony] + + '@rolldown/binding-wasm32-wasi@1.0.0-rc.15': + resolution: {integrity: sha512-ApLruZq/ig+nhaE7OJm4lDjayUnOHVUa77zGeqnqZ9pn0ovdVbbNPerVibLXDmWeUZXjIYIT8V3xkT58Rm9u5Q==} + engines: {node: '>=14.0.0'} + cpu: [wasm32] + + '@rolldown/binding-win32-arm64-msvc@1.0.0-rc.15': + resolution: {integrity: sha512-KmoUoU7HnN+Si5YWJigfTws1jz1bKBYDQKdbLspz0UaqjjFkddHsqorgiW1mxcAj88lYUE6NC/zJNwT+SloqtA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [win32] + + '@rolldown/binding-win32-x64-msvc@1.0.0-rc.15': + resolution: {integrity: sha512-3P2A8L+x75qavWLe/Dll3EYBJLQmtkJN8rfh+U/eR3MqMgL/h98PhYI+JFfXuDPgPeCB7iZAKiqii5vqOvnA0g==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [win32] + + '@rolldown/pluginutils@1.0.0-rc.15': + resolution: {integrity: sha512-UromN0peaE53IaBRe9W7CjrZgXl90fqGpK+mIZbA3qSTeYqg3pqpROBdIPvOG3F5ereDHNwoHBI2e50n1BDr1g==} + + '@rolldown/pluginutils@1.0.0-rc.7': + resolution: {integrity: sha512-qujRfC8sFVInYSPPMLQByRh7zhwkGFS4+tyMQ83srV1qrxL4g8E2tyxVVyxd0+8QeBM1mIk9KbWxkegRr76XzA==} + + '@tybys/wasm-util@0.10.1': + resolution: {integrity: sha512-9tTaPJLSiejZKx+Bmog4uSubteqTvFrVrURwkmHixBo0G4seD0zUxp98E1DzUBJxLQ3NPwXrGKDiVjwx/DpPsg==} + + '@types/estree@1.0.8': + resolution: {integrity: sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==} + + '@types/json-schema@7.0.15': + resolution: {integrity: sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==} + + '@types/node@24.12.2': + resolution: {integrity: sha512-A1sre26ke7HDIuY/M23nd9gfB+nrmhtYyMINbjI1zHJxYteKR6qSMX56FsmjMcDb3SMcjJg5BiRRgOCC/yBD0g==} + + '@types/react-dom@19.2.3': + resolution: {integrity: sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ==} + peerDependencies: + '@types/react': ^19.2.0 + + '@types/react@19.2.14': + resolution: {integrity: sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w==} + + '@typescript-eslint/eslint-plugin@8.58.2': + resolution: {integrity: sha512-aC2qc5thQahutKjP+cl8cgN9DWe3ZUqVko30CMSZHnFEHyhOYoZSzkGtAI2mcwZ38xeImDucI4dnqsHiOYuuCw==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + '@typescript-eslint/parser': ^8.58.2 + eslint: ^8.57.0 || ^9.0.0 || ^10.0.0 + typescript: '>=4.8.4 <6.1.0' + + '@typescript-eslint/parser@8.58.2': + resolution: {integrity: sha512-/Zb/xaIDfxeJnvishjGdcR4jmr7S+bda8PKNhRGdljDM+elXhlvN0FyPSsMnLmJUrVG9aPO6dof80wjMawsASg==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + eslint: ^8.57.0 || ^9.0.0 || ^10.0.0 + typescript: '>=4.8.4 <6.1.0' + + '@typescript-eslint/project-service@8.58.2': + resolution: {integrity: sha512-Cq6UfpZZk15+r87BkIh5rDpi38W4b+Sjnb8wQCPPDDweS/LRCFjCyViEbzHk5Ck3f2QDfgmlxqSa7S7clDtlfg==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + typescript: '>=4.8.4 <6.1.0' + + '@typescript-eslint/scope-manager@8.58.2': + resolution: {integrity: sha512-SgmyvDPexWETQek+qzZnrG6844IaO02UVyOLhI4wpo82dpZJY9+6YZCKAMFzXb7qhx37mFK1QcPQ18tud+vo6Q==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@typescript-eslint/tsconfig-utils@8.58.2': + resolution: {integrity: sha512-3SR+RukipDvkkKp/d0jP0dyzuls3DbGmwDpVEc5wqk5f38KFThakqAAO0XMirWAE+kT00oTauTbzMFGPoAzB0A==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + typescript: '>=4.8.4 <6.1.0' + + '@typescript-eslint/type-utils@8.58.2': + resolution: {integrity: sha512-Z7EloNR/B389FvabdGeTo2XMs4W9TjtPiO9DAsmT0yom0bwlPyRjkJ1uCdW1DvrrrYP50AJZ9Xc3sByZA9+dcg==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + eslint: ^8.57.0 || ^9.0.0 || ^10.0.0 + typescript: '>=4.8.4 <6.1.0' + + '@typescript-eslint/types@8.58.2': + resolution: {integrity: sha512-9TukXyATBQf/Jq9AMQXfvurk+G5R2MwfqQGDR2GzGz28HvY/lXNKGhkY+6IOubwcquikWk5cjlgPvD2uAA7htQ==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@typescript-eslint/typescript-estree@8.58.2': + resolution: {integrity: sha512-ELGuoofuhhoCvNbQjFFiobFcGgcDCEm0ThWdmO4Z0UzLqPXS3KFvnEZ+SHewwOYHjM09tkzOWXNTv9u6Gqtyuw==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + typescript: '>=4.8.4 <6.1.0' + + '@typescript-eslint/utils@8.58.2': + resolution: {integrity: sha512-QZfjHNEzPY8+l0+fIXMvuQ2sJlplB4zgDZvA+NmvZsZv3EQwOcc1DuIU1VJUTWZ/RKouBMhDyNaBMx4sWvrzRA==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + eslint: ^8.57.0 || ^9.0.0 || ^10.0.0 + typescript: '>=4.8.4 <6.1.0' + + '@typescript-eslint/visitor-keys@8.58.2': + resolution: {integrity: sha512-f1WO2Lx8a9t8DARmcWAUPJbu0G20bJlj8L4z72K00TMeJAoyLr/tHhI/pzYBLrR4dXWkcxO1cWYZEOX8DKHTqA==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@vitejs/plugin-react@6.0.1': + resolution: {integrity: sha512-l9X/E3cDb+xY3SWzlG1MOGt2usfEHGMNIaegaUGFsLkb3RCn/k8/TOXBcab+OndDI4TBtktT8/9BwwW8Vi9KUQ==} + engines: {node: ^20.19.0 || >=22.12.0} + peerDependencies: + '@rolldown/plugin-babel': ^0.1.7 || ^0.2.0 + babel-plugin-react-compiler: ^1.0.0 + vite: ^8.0.0 + peerDependenciesMeta: + '@rolldown/plugin-babel': + optional: true + babel-plugin-react-compiler: + optional: true + + acorn-jsx@5.3.2: + resolution: {integrity: sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==} + peerDependencies: + acorn: ^6.0.0 || ^7.0.0 || ^8.0.0 + + acorn@8.16.0: + resolution: {integrity: sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==} + engines: {node: '>=0.4.0'} + hasBin: true + + ajv@6.14.0: + resolution: {integrity: sha512-IWrosm/yrn43eiKqkfkHis7QioDleaXQHdDVPKg0FSwwd/DuvyX79TZnFOnYpB7dcsFAMmtFztZuXPDvSePkFw==} + + ansi-styles@4.3.0: + resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==} + engines: {node: '>=8'} + + argparse@2.0.1: + resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} + + balanced-match@1.0.2: + resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} + + balanced-match@4.0.4: + resolution: {integrity: sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==} + engines: {node: 18 || 20 || >=22} + + baseline-browser-mapping@2.10.19: + resolution: {integrity: sha512-qCkNLi2sfBOn8XhZQ0FXsT1Ki/Yo5P90hrkRamVFRS7/KV9hpfA4HkoWNU152+8w0zPjnxo5psx5NL3PSGgv5g==} + engines: {node: '>=6.0.0'} + hasBin: true + + brace-expansion@1.1.14: + resolution: {integrity: sha512-MWPGfDxnyzKU7rNOW9SP/c50vi3xrmrua/+6hfPbCS2ABNWfx24vPidzvC7krjU/RTo235sV776ymlsMtGKj8g==} + + brace-expansion@5.0.5: + resolution: {integrity: sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==} + engines: {node: 18 || 20 || >=22} + + browserslist@4.28.2: + resolution: {integrity: sha512-48xSriZYYg+8qXna9kwqjIVzuQxi+KYWp2+5nCYnYKPTr0LvD89Jqk2Or5ogxz0NUMfIjhh2lIUX/LyX9B4oIg==} + engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7} + hasBin: true + + callsites@3.1.0: + resolution: {integrity: sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==} + engines: {node: '>=6'} + + caniuse-lite@1.0.30001788: + resolution: {integrity: sha512-6q8HFp+lOQtcf7wBK+uEenxymVWkGKkjFpCvw5W25cmMwEDU45p1xQFBQv8JDlMMry7eNxyBaR+qxgmTUZkIRQ==} + + chalk@4.1.2: + resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} + engines: {node: '>=10'} + + color-convert@2.0.1: + resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} + engines: {node: '>=7.0.0'} + + color-name@1.1.4: + resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} + + concat-map@0.0.1: + resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==} + + convert-source-map@2.0.0: + resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==} + + cross-spawn@7.0.6: + resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==} + engines: {node: '>= 8'} + + csstype@3.2.3: + resolution: {integrity: sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==} + + debug@4.4.3: + resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + + deep-is@0.1.4: + resolution: {integrity: sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==} + + detect-libc@2.1.2: + resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} + engines: {node: '>=8'} + + electron-to-chromium@1.5.337: + resolution: {integrity: sha512-15gKW9mRUNP9RdzhedJNypFUxtYWSXohFz2nTLzM272xbRXHws68kNDzyATG3qej+vUj/7Sn9hf5XTDh0XK6/w==} + + escalade@3.2.0: + resolution: {integrity: sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==} + engines: {node: '>=6'} + + escape-string-regexp@4.0.0: + resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==} + engines: {node: '>=10'} + + eslint-plugin-react-hooks@7.0.1: + resolution: {integrity: sha512-O0d0m04evaNzEPoSW+59Mezf8Qt0InfgGIBJnpC0h3NH/WjUAR7BIKUfysC6todmtiZ/A0oUVS8Gce0WhBrHsA==} + engines: {node: '>=18'} + peerDependencies: + eslint: ^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0-0 || ^9.0.0 + + eslint-plugin-react-refresh@0.5.2: + resolution: {integrity: sha512-hmgTH57GfzoTFjVN0yBwTggnsVUF2tcqi7RJZHqi9lIezSs4eFyAMktA68YD4r5kNw1mxyY4dmkyoFDb3FIqrA==} + peerDependencies: + eslint: ^9 || ^10 + + eslint-scope@8.4.0: + resolution: {integrity: sha512-sNXOfKCn74rt8RICKMvJS7XKV/Xk9kA7DyJr8mJik3S7Cwgy3qlkkmyS2uQB3jiJg6VNdZd/pDBJu0nvG2NlTg==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + eslint-visitor-keys@3.4.3: + resolution: {integrity: sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + + eslint-visitor-keys@4.2.1: + resolution: {integrity: sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + eslint-visitor-keys@5.0.1: + resolution: {integrity: sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==} + engines: {node: ^20.19.0 || ^22.13.0 || >=24} + + eslint@9.39.4: + resolution: {integrity: sha512-XoMjdBOwe/esVgEvLmNsD3IRHkm7fbKIUGvrleloJXUZgDHig2IPWNniv+GwjyJXzuNqVjlr5+4yVUZjycJwfQ==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + hasBin: true + peerDependencies: + jiti: '*' + peerDependenciesMeta: + jiti: + optional: true + + espree@10.4.0: + resolution: {integrity: sha512-j6PAQ2uUr79PZhBjP5C5fhl8e39FmRnOjsD5lGnWrFU8i2G776tBK7+nP8KuQUTTyAZUwfQqXAgrVH5MbH9CYQ==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + esquery@1.7.0: + resolution: {integrity: sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g==} + engines: {node: '>=0.10'} + + esrecurse@4.3.0: + resolution: {integrity: sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==} + engines: {node: '>=4.0'} + + estraverse@5.3.0: + resolution: {integrity: sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==} + engines: {node: '>=4.0'} + + esutils@2.0.3: + resolution: {integrity: sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==} + engines: {node: '>=0.10.0'} + + fast-deep-equal@3.1.3: + resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} + + fast-json-stable-stringify@2.1.0: + resolution: {integrity: sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==} + + fast-levenshtein@2.0.6: + resolution: {integrity: sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==} + + fdir@6.5.0: + resolution: {integrity: sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==} + engines: {node: '>=12.0.0'} + peerDependencies: + picomatch: ^3 || ^4 + peerDependenciesMeta: + picomatch: + optional: true + + file-entry-cache@8.0.0: + resolution: {integrity: sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==} + engines: {node: '>=16.0.0'} + + find-up@5.0.0: + resolution: {integrity: sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==} + engines: {node: '>=10'} + + flat-cache@4.0.1: + resolution: {integrity: sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==} + engines: {node: '>=16'} + + flatted@3.4.2: + resolution: {integrity: sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA==} + + fsevents@2.3.3: + resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + + gensync@1.0.0-beta.2: + resolution: {integrity: sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==} + engines: {node: '>=6.9.0'} + + glob-parent@6.0.2: + resolution: {integrity: sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==} + engines: {node: '>=10.13.0'} + + globals@14.0.0: + resolution: {integrity: sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ==} + engines: {node: '>=18'} + + globals@17.5.0: + resolution: {integrity: sha512-qoV+HK2yFl/366t2/Cb3+xxPUo5BuMynomoDmiaZBIdbs+0pYbjfZU+twLhGKp4uCZ/+NbtpVepH5bGCxRyy2g==} + engines: {node: '>=18'} + + has-flag@4.0.0: + resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} + engines: {node: '>=8'} + + hermes-estree@0.25.1: + resolution: {integrity: sha512-0wUoCcLp+5Ev5pDW2OriHC2MJCbwLwuRx+gAqMTOkGKJJiBCLjtrvy4PWUGn6MIVefecRpzoOZ/UV6iGdOr+Cw==} + + hermes-parser@0.25.1: + resolution: {integrity: sha512-6pEjquH3rqaI6cYAXYPcz9MS4rY6R4ngRgrgfDshRptUZIc3lw0MCIJIGDj9++mfySOuPTHB4nrSW99BCvOPIA==} + + ignore@5.3.2: + resolution: {integrity: sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==} + engines: {node: '>= 4'} + + ignore@7.0.5: + resolution: {integrity: sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==} + engines: {node: '>= 4'} + + import-fresh@3.3.1: + resolution: {integrity: sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==} + engines: {node: '>=6'} + + imurmurhash@0.1.4: + resolution: {integrity: sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==} + engines: {node: '>=0.8.19'} + + is-extglob@2.1.1: + resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==} + engines: {node: '>=0.10.0'} + + is-glob@4.0.3: + resolution: {integrity: sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==} + engines: {node: '>=0.10.0'} + + isexe@2.0.0: + resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} + + js-tokens@4.0.0: + resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} + + js-yaml@4.1.1: + resolution: {integrity: sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==} + hasBin: true + + jsesc@3.1.0: + resolution: {integrity: sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==} + engines: {node: '>=6'} + hasBin: true + + json-buffer@3.0.1: + resolution: {integrity: sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==} + + json-schema-traverse@0.4.1: + resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==} + + json-stable-stringify-without-jsonify@1.0.1: + resolution: {integrity: sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==} + + json5@2.2.3: + resolution: {integrity: sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==} + engines: {node: '>=6'} + hasBin: true + + keyv@4.5.4: + resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} + + levn@0.4.1: + resolution: {integrity: sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==} + engines: {node: '>= 0.8.0'} + + lightningcss-android-arm64@1.32.0: + resolution: {integrity: sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [android] + + lightningcss-darwin-arm64@1.32.0: + resolution: {integrity: sha512-RzeG9Ju5bag2Bv1/lwlVJvBE3q6TtXskdZLLCyfg5pt+HLz9BqlICO7LZM7VHNTTn/5PRhHFBSjk5lc4cmscPQ==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [darwin] + + lightningcss-darwin-x64@1.32.0: + resolution: {integrity: sha512-U+QsBp2m/s2wqpUYT/6wnlagdZbtZdndSmut/NJqlCcMLTWp5muCrID+K5UJ6jqD2BFshejCYXniPDbNh73V8w==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [darwin] + + lightningcss-freebsd-x64@1.32.0: + resolution: {integrity: sha512-JCTigedEksZk3tHTTthnMdVfGf61Fky8Ji2E4YjUTEQX14xiy/lTzXnu1vwiZe3bYe0q+SpsSH/CTeDXK6WHig==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [freebsd] + + lightningcss-linux-arm-gnueabihf@1.32.0: + resolution: {integrity: sha512-x6rnnpRa2GL0zQOkt6rts3YDPzduLpWvwAF6EMhXFVZXD4tPrBkEFqzGowzCsIWsPjqSK+tyNEODUBXeeVHSkw==} + engines: {node: '>= 12.0.0'} + cpu: [arm] + os: [linux] + + lightningcss-linux-arm64-gnu@1.32.0: + resolution: {integrity: sha512-0nnMyoyOLRJXfbMOilaSRcLH3Jw5z9HDNGfT/gwCPgaDjnx0i8w7vBzFLFR1f6CMLKF8gVbebmkUN3fa/kQJpQ==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [linux] + libc: [glibc] + + lightningcss-linux-arm64-musl@1.32.0: + resolution: {integrity: sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [linux] + libc: [musl] + + lightningcss-linux-x64-gnu@1.32.0: + resolution: {integrity: sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [linux] + libc: [glibc] + + lightningcss-linux-x64-musl@1.32.0: + resolution: {integrity: sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [linux] + libc: [musl] + + lightningcss-win32-arm64-msvc@1.32.0: + resolution: {integrity: sha512-8SbC8BR40pS6baCM8sbtYDSwEVQd4JlFTOlaD3gWGHfThTcABnNDBda6eTZeqbofalIJhFx0qKzgHJmcPTnGdw==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [win32] + + lightningcss-win32-x64-msvc@1.32.0: + resolution: {integrity: sha512-Amq9B/SoZYdDi1kFrojnoqPLxYhQ4Wo5XiL8EVJrVsB8ARoC1PWW6VGtT0WKCemjy8aC+louJnjS7U18x3b06Q==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [win32] + + lightningcss@1.32.0: + resolution: {integrity: sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ==} + engines: {node: '>= 12.0.0'} + + locate-path@6.0.0: + resolution: {integrity: sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==} + engines: {node: '>=10'} + + lodash.merge@4.6.2: + resolution: {integrity: sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==} + + lru-cache@5.1.1: + resolution: {integrity: sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==} + + minimatch@10.2.5: + resolution: {integrity: sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==} + engines: {node: 18 || 20 || >=22} + + minimatch@3.1.5: + resolution: {integrity: sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==} + + ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + + nanoid@3.3.11: + resolution: {integrity: sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==} + engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} + hasBin: true + + natural-compare@1.4.0: + resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} + + node-releases@2.0.37: + resolution: {integrity: sha512-1h5gKZCF+pO/o3Iqt5Jp7wc9rH3eJJ0+nh/CIoiRwjRxde/hAHyLPXYN4V3CqKAbiZPSeJFSWHmJsbkicta0Eg==} + + optionator@0.9.4: + resolution: {integrity: sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==} + engines: {node: '>= 0.8.0'} + + p-limit@3.1.0: + resolution: {integrity: sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==} + engines: {node: '>=10'} + + p-locate@5.0.0: + resolution: {integrity: sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==} + engines: {node: '>=10'} + + parent-module@1.0.1: + resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==} + engines: {node: '>=6'} + + path-exists@4.0.0: + resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==} + engines: {node: '>=8'} + + path-key@3.1.1: + resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} + engines: {node: '>=8'} + + picocolors@1.1.1: + resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} + + picomatch@4.0.4: + resolution: {integrity: sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==} + engines: {node: '>=12'} + + postcss@8.5.10: + resolution: {integrity: sha512-pMMHxBOZKFU6HgAZ4eyGnwXF/EvPGGqUr0MnZ5+99485wwW41kW91A4LOGxSHhgugZmSChL5AlElNdwlNgcnLQ==} + engines: {node: ^10 || ^12 || >=14} + + prelude-ls@1.2.1: + resolution: {integrity: sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==} + engines: {node: '>= 0.8.0'} + + punycode@2.3.1: + resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} + engines: {node: '>=6'} + + react-dom@19.2.5: + resolution: {integrity: sha512-J5bAZz+DXMMwW/wV3xzKke59Af6CHY7G4uYLN1OvBcKEsWOs4pQExj86BBKamxl/Ik5bx9whOrvBlSDfWzgSag==} + peerDependencies: + react: ^19.2.5 + + react@19.2.5: + resolution: {integrity: sha512-llUJLzz1zTUBrskt2pwZgLq59AemifIftw4aB7JxOqf1HY2FDaGDxgwpAPVzHU1kdWabH7FauP4i1oEeer2WCA==} + engines: {node: '>=0.10.0'} + + resolve-from@4.0.0: + resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==} + engines: {node: '>=4'} + + rolldown@1.0.0-rc.15: + resolution: {integrity: sha512-Ff31guA5zT6WjnGp0SXw76X6hzGRk/OQq2hE+1lcDe+lJdHSgnSX6nK3erbONHyCbpSj9a9E+uX/OvytZoWp2g==} + engines: {node: ^20.19.0 || >=22.12.0} + hasBin: true + + scheduler@0.27.0: + resolution: {integrity: sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q==} + + semver@6.3.1: + resolution: {integrity: sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==} + hasBin: true + + semver@7.7.4: + resolution: {integrity: sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==} + engines: {node: '>=10'} + hasBin: true + + shebang-command@2.0.0: + resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==} + engines: {node: '>=8'} + + shebang-regex@3.0.0: + resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==} + engines: {node: '>=8'} + + source-map-js@1.2.1: + resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} + engines: {node: '>=0.10.0'} + + strip-json-comments@3.1.1: + resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==} + engines: {node: '>=8'} + + supports-color@7.2.0: + resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} + engines: {node: '>=8'} + + tinyglobby@0.2.16: + resolution: {integrity: sha512-pn99VhoACYR8nFHhxqix+uvsbXineAasWm5ojXoN8xEwK5Kd3/TrhNn1wByuD52UxWRLy8pu+kRMniEi6Eq9Zg==} + engines: {node: '>=12.0.0'} + + ts-api-utils@2.5.0: + resolution: {integrity: sha512-OJ/ibxhPlqrMM0UiNHJ/0CKQkoKF243/AEmplt3qpRgkW8VG7IfOS41h7V8TjITqdByHzrjcS/2si+y4lIh8NA==} + engines: {node: '>=18.12'} + peerDependencies: + typescript: '>=4.8.4' + + tslib@2.8.1: + resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + + type-check@0.4.0: + resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==} + engines: {node: '>= 0.8.0'} + + typescript-eslint@8.58.2: + resolution: {integrity: sha512-V8iSng9mRbdZjl54VJ9NKr6ZB+dW0J3TzRXRGcSbLIej9jV86ZRtlYeTKDR/QLxXykocJ5icNzbsl2+5TzIvcQ==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + eslint: ^8.57.0 || ^9.0.0 || ^10.0.0 + typescript: '>=4.8.4 <6.1.0' + + typescript@6.0.2: + resolution: {integrity: sha512-bGdAIrZ0wiGDo5l8c++HWtbaNCWTS4UTv7RaTH/ThVIgjkveJt83m74bBHMJkuCbslY8ixgLBVZJIOiQlQTjfQ==} + engines: {node: '>=14.17'} + hasBin: true + + undici-types@7.16.0: + resolution: {integrity: sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==} + + update-browserslist-db@1.2.3: + resolution: {integrity: sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==} + hasBin: true + peerDependencies: + browserslist: '>= 4.21.0' + + uri-js@4.4.1: + resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} + + vite@8.0.8: + resolution: {integrity: sha512-dbU7/iLVa8KZALJyLOBOQ88nOXtNG8vxKuOT4I2mD+Ya70KPceF4IAmDsmU0h1Qsn5bPrvsY9HJstCRh3hG6Uw==} + engines: {node: ^20.19.0 || >=22.12.0} + hasBin: true + peerDependencies: + '@types/node': ^20.19.0 || >=22.12.0 + '@vitejs/devtools': ^0.1.0 + esbuild: ^0.27.0 || ^0.28.0 + jiti: '>=1.21.0' + less: ^4.0.0 + sass: ^1.70.0 + sass-embedded: ^1.70.0 + stylus: '>=0.54.8' + sugarss: ^5.0.0 + terser: ^5.16.0 + tsx: ^4.8.1 + yaml: ^2.4.2 + peerDependenciesMeta: + '@types/node': + optional: true + '@vitejs/devtools': + optional: true + esbuild: + optional: true + jiti: + optional: true + less: + optional: true + sass: + optional: true + sass-embedded: + optional: true + stylus: + optional: true + sugarss: + optional: true + terser: + optional: true + tsx: + optional: true + yaml: + optional: true + + which@2.0.2: + resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==} + engines: {node: '>= 8'} + hasBin: true + + word-wrap@1.2.5: + resolution: {integrity: sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==} + engines: {node: '>=0.10.0'} + + yallist@3.1.1: + resolution: {integrity: sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==} + + yocto-queue@0.1.0: + resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} + engines: {node: '>=10'} + + zod-validation-error@4.0.2: + resolution: {integrity: sha512-Q6/nZLe6jxuU80qb/4uJ4t5v2VEZ44lzQjPDhYJNztRQ4wyWc6VF3D3Kb/fAuPetZQnhS3hnajCf9CsWesghLQ==} + engines: {node: '>=18.0.0'} + peerDependencies: + zod: ^3.25.0 || ^4.0.0 + + zod@4.3.6: + resolution: {integrity: sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==} + +snapshots: + + '@babel/code-frame@7.29.0': + dependencies: + '@babel/helper-validator-identifier': 7.28.5 + js-tokens: 4.0.0 + picocolors: 1.1.1 + + '@babel/compat-data@7.29.0': {} + + '@babel/core@7.29.0': + dependencies: + '@babel/code-frame': 7.29.0 + '@babel/generator': 7.29.1 + '@babel/helper-compilation-targets': 7.28.6 + '@babel/helper-module-transforms': 7.28.6(@babel/core@7.29.0) + '@babel/helpers': 7.29.2 + '@babel/parser': 7.29.2 + '@babel/template': 7.28.6 + '@babel/traverse': 7.29.0 + '@babel/types': 7.29.0 + '@jridgewell/remapping': 2.3.5 + convert-source-map: 2.0.0 + debug: 4.4.3 + gensync: 1.0.0-beta.2 + json5: 2.2.3 + semver: 6.3.1 + transitivePeerDependencies: + - supports-color + + '@babel/generator@7.29.1': + dependencies: + '@babel/parser': 7.29.2 + '@babel/types': 7.29.0 + '@jridgewell/gen-mapping': 0.3.13 + '@jridgewell/trace-mapping': 0.3.31 + jsesc: 3.1.0 + + '@babel/helper-compilation-targets@7.28.6': + dependencies: + '@babel/compat-data': 7.29.0 + '@babel/helper-validator-option': 7.27.1 + browserslist: 4.28.2 + lru-cache: 5.1.1 + semver: 6.3.1 + + '@babel/helper-globals@7.28.0': {} + + '@babel/helper-module-imports@7.28.6': + dependencies: + '@babel/traverse': 7.29.0 + '@babel/types': 7.29.0 + transitivePeerDependencies: + - supports-color + + '@babel/helper-module-transforms@7.28.6(@babel/core@7.29.0)': + dependencies: + '@babel/core': 7.29.0 + '@babel/helper-module-imports': 7.28.6 + '@babel/helper-validator-identifier': 7.28.5 + '@babel/traverse': 7.29.0 + transitivePeerDependencies: + - supports-color + + '@babel/helper-string-parser@7.27.1': {} + + '@babel/helper-validator-identifier@7.28.5': {} + + '@babel/helper-validator-option@7.27.1': {} + + '@babel/helpers@7.29.2': + dependencies: + '@babel/template': 7.28.6 + '@babel/types': 7.29.0 + + '@babel/parser@7.29.2': + dependencies: + '@babel/types': 7.29.0 + + '@babel/template@7.28.6': + dependencies: + '@babel/code-frame': 7.29.0 + '@babel/parser': 7.29.2 + '@babel/types': 7.29.0 + + '@babel/traverse@7.29.0': + dependencies: + '@babel/code-frame': 7.29.0 + '@babel/generator': 7.29.1 + '@babel/helper-globals': 7.28.0 + '@babel/parser': 7.29.2 + '@babel/template': 7.28.6 + '@babel/types': 7.29.0 + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + + '@babel/types@7.29.0': + dependencies: + '@babel/helper-string-parser': 7.27.1 + '@babel/helper-validator-identifier': 7.28.5 + + '@emnapi/core@1.9.2': + dependencies: + '@emnapi/wasi-threads': 1.2.1 + tslib: 2.8.1 + optional: true + + '@emnapi/runtime@1.9.2': + dependencies: + tslib: 2.8.1 + optional: true + + '@emnapi/wasi-threads@1.2.1': + dependencies: + tslib: 2.8.1 + optional: true + + '@eslint-community/eslint-utils@4.9.1(eslint@9.39.4)': + dependencies: + eslint: 9.39.4 + eslint-visitor-keys: 3.4.3 + + '@eslint-community/regexpp@4.12.2': {} + + '@eslint/config-array@0.21.2': + dependencies: + '@eslint/object-schema': 2.1.7 + debug: 4.4.3 + minimatch: 3.1.5 + transitivePeerDependencies: + - supports-color + + '@eslint/config-helpers@0.4.2': + dependencies: + '@eslint/core': 0.17.0 + + '@eslint/core@0.17.0': + dependencies: + '@types/json-schema': 7.0.15 + + '@eslint/eslintrc@3.3.5': + dependencies: + ajv: 6.14.0 + debug: 4.4.3 + espree: 10.4.0 + globals: 14.0.0 + ignore: 5.3.2 + import-fresh: 3.3.1 + js-yaml: 4.1.1 + minimatch: 3.1.5 + strip-json-comments: 3.1.1 + transitivePeerDependencies: + - supports-color + + '@eslint/js@9.39.4': {} + + '@eslint/object-schema@2.1.7': {} + + '@eslint/plugin-kit@0.4.1': + dependencies: + '@eslint/core': 0.17.0 + levn: 0.4.1 + + '@humanfs/core@0.19.1': {} + + '@humanfs/node@0.16.7': + dependencies: + '@humanfs/core': 0.19.1 + '@humanwhocodes/retry': 0.4.3 + + '@humanwhocodes/module-importer@1.0.1': {} + + '@humanwhocodes/retry@0.4.3': {} + + '@jridgewell/gen-mapping@0.3.13': + dependencies: + '@jridgewell/sourcemap-codec': 1.5.5 + '@jridgewell/trace-mapping': 0.3.31 + + '@jridgewell/remapping@2.3.5': + dependencies: + '@jridgewell/gen-mapping': 0.3.13 + '@jridgewell/trace-mapping': 0.3.31 + + '@jridgewell/resolve-uri@3.1.2': {} + + '@jridgewell/sourcemap-codec@1.5.5': {} + + '@jridgewell/trace-mapping@0.3.31': + dependencies: + '@jridgewell/resolve-uri': 3.1.2 + '@jridgewell/sourcemap-codec': 1.5.5 + + '@mediapipe/tasks-vision@0.10.34': {} + + '@napi-rs/wasm-runtime@1.1.4(@emnapi/core@1.9.2)(@emnapi/runtime@1.9.2)': + dependencies: + '@emnapi/core': 1.9.2 + '@emnapi/runtime': 1.9.2 + '@tybys/wasm-util': 0.10.1 + optional: true + + '@oxc-project/types@0.124.0': {} + + '@rolldown/binding-android-arm64@1.0.0-rc.15': + optional: true + + '@rolldown/binding-darwin-arm64@1.0.0-rc.15': + optional: true + + '@rolldown/binding-darwin-x64@1.0.0-rc.15': + optional: true + + '@rolldown/binding-freebsd-x64@1.0.0-rc.15': + optional: true + + '@rolldown/binding-linux-arm-gnueabihf@1.0.0-rc.15': + optional: true + + '@rolldown/binding-linux-arm64-gnu@1.0.0-rc.15': + optional: true + + '@rolldown/binding-linux-arm64-musl@1.0.0-rc.15': + optional: true + + '@rolldown/binding-linux-ppc64-gnu@1.0.0-rc.15': + optional: true + + '@rolldown/binding-linux-s390x-gnu@1.0.0-rc.15': + optional: true + + '@rolldown/binding-linux-x64-gnu@1.0.0-rc.15': + optional: true + + '@rolldown/binding-linux-x64-musl@1.0.0-rc.15': + optional: true + + '@rolldown/binding-openharmony-arm64@1.0.0-rc.15': + optional: true + + '@rolldown/binding-wasm32-wasi@1.0.0-rc.15': + dependencies: + '@emnapi/core': 1.9.2 + '@emnapi/runtime': 1.9.2 + '@napi-rs/wasm-runtime': 1.1.4(@emnapi/core@1.9.2)(@emnapi/runtime@1.9.2) + optional: true + + '@rolldown/binding-win32-arm64-msvc@1.0.0-rc.15': + optional: true + + '@rolldown/binding-win32-x64-msvc@1.0.0-rc.15': + optional: true + + '@rolldown/pluginutils@1.0.0-rc.15': {} + + '@rolldown/pluginutils@1.0.0-rc.7': {} + + '@tybys/wasm-util@0.10.1': + dependencies: + tslib: 2.8.1 + optional: true + + '@types/estree@1.0.8': {} + + '@types/json-schema@7.0.15': {} + + '@types/node@24.12.2': + dependencies: + undici-types: 7.16.0 + + '@types/react-dom@19.2.3(@types/react@19.2.14)': + dependencies: + '@types/react': 19.2.14 + + '@types/react@19.2.14': + dependencies: + csstype: 3.2.3 + + '@typescript-eslint/eslint-plugin@8.58.2(@typescript-eslint/parser@8.58.2(eslint@9.39.4)(typescript@6.0.2))(eslint@9.39.4)(typescript@6.0.2)': + dependencies: + '@eslint-community/regexpp': 4.12.2 + '@typescript-eslint/parser': 8.58.2(eslint@9.39.4)(typescript@6.0.2) + '@typescript-eslint/scope-manager': 8.58.2 + '@typescript-eslint/type-utils': 8.58.2(eslint@9.39.4)(typescript@6.0.2) + '@typescript-eslint/utils': 8.58.2(eslint@9.39.4)(typescript@6.0.2) + '@typescript-eslint/visitor-keys': 8.58.2 + eslint: 9.39.4 + ignore: 7.0.5 + natural-compare: 1.4.0 + ts-api-utils: 2.5.0(typescript@6.0.2) + typescript: 6.0.2 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/parser@8.58.2(eslint@9.39.4)(typescript@6.0.2)': + dependencies: + '@typescript-eslint/scope-manager': 8.58.2 + '@typescript-eslint/types': 8.58.2 + '@typescript-eslint/typescript-estree': 8.58.2(typescript@6.0.2) + '@typescript-eslint/visitor-keys': 8.58.2 + debug: 4.4.3 + eslint: 9.39.4 + typescript: 6.0.2 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/project-service@8.58.2(typescript@6.0.2)': + dependencies: + '@typescript-eslint/tsconfig-utils': 8.58.2(typescript@6.0.2) + '@typescript-eslint/types': 8.58.2 + debug: 4.4.3 + typescript: 6.0.2 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/scope-manager@8.58.2': + dependencies: + '@typescript-eslint/types': 8.58.2 + '@typescript-eslint/visitor-keys': 8.58.2 + + '@typescript-eslint/tsconfig-utils@8.58.2(typescript@6.0.2)': + dependencies: + typescript: 6.0.2 + + '@typescript-eslint/type-utils@8.58.2(eslint@9.39.4)(typescript@6.0.2)': + dependencies: + '@typescript-eslint/types': 8.58.2 + '@typescript-eslint/typescript-estree': 8.58.2(typescript@6.0.2) + '@typescript-eslint/utils': 8.58.2(eslint@9.39.4)(typescript@6.0.2) + debug: 4.4.3 + eslint: 9.39.4 + ts-api-utils: 2.5.0(typescript@6.0.2) + typescript: 6.0.2 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/types@8.58.2': {} + + '@typescript-eslint/typescript-estree@8.58.2(typescript@6.0.2)': + dependencies: + '@typescript-eslint/project-service': 8.58.2(typescript@6.0.2) + '@typescript-eslint/tsconfig-utils': 8.58.2(typescript@6.0.2) + '@typescript-eslint/types': 8.58.2 + '@typescript-eslint/visitor-keys': 8.58.2 + debug: 4.4.3 + minimatch: 10.2.5 + semver: 7.7.4 + tinyglobby: 0.2.16 + ts-api-utils: 2.5.0(typescript@6.0.2) + typescript: 6.0.2 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/utils@8.58.2(eslint@9.39.4)(typescript@6.0.2)': + dependencies: + '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.4) + '@typescript-eslint/scope-manager': 8.58.2 + '@typescript-eslint/types': 8.58.2 + '@typescript-eslint/typescript-estree': 8.58.2(typescript@6.0.2) + eslint: 9.39.4 + typescript: 6.0.2 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/visitor-keys@8.58.2': + dependencies: + '@typescript-eslint/types': 8.58.2 + eslint-visitor-keys: 5.0.1 + + '@vitejs/plugin-react@6.0.1(vite@8.0.8(@types/node@24.12.2))': + dependencies: + '@rolldown/pluginutils': 1.0.0-rc.7 + vite: 8.0.8(@types/node@24.12.2) + + acorn-jsx@5.3.2(acorn@8.16.0): + dependencies: + acorn: 8.16.0 + + acorn@8.16.0: {} + + ajv@6.14.0: + dependencies: + fast-deep-equal: 3.1.3 + fast-json-stable-stringify: 2.1.0 + json-schema-traverse: 0.4.1 + uri-js: 4.4.1 + + ansi-styles@4.3.0: + dependencies: + color-convert: 2.0.1 + + argparse@2.0.1: {} + + balanced-match@1.0.2: {} + + balanced-match@4.0.4: {} + + baseline-browser-mapping@2.10.19: {} + + brace-expansion@1.1.14: + dependencies: + balanced-match: 1.0.2 + concat-map: 0.0.1 + + brace-expansion@5.0.5: + dependencies: + balanced-match: 4.0.4 + + browserslist@4.28.2: + dependencies: + baseline-browser-mapping: 2.10.19 + caniuse-lite: 1.0.30001788 + electron-to-chromium: 1.5.337 + node-releases: 2.0.37 + update-browserslist-db: 1.2.3(browserslist@4.28.2) + + callsites@3.1.0: {} + + caniuse-lite@1.0.30001788: {} + + chalk@4.1.2: + dependencies: + ansi-styles: 4.3.0 + supports-color: 7.2.0 + + color-convert@2.0.1: + dependencies: + color-name: 1.1.4 + + color-name@1.1.4: {} + + concat-map@0.0.1: {} + + convert-source-map@2.0.0: {} + + cross-spawn@7.0.6: + dependencies: + path-key: 3.1.1 + shebang-command: 2.0.0 + which: 2.0.2 + + csstype@3.2.3: {} + + debug@4.4.3: + dependencies: + ms: 2.1.3 + + deep-is@0.1.4: {} + + detect-libc@2.1.2: {} + + electron-to-chromium@1.5.337: {} + + escalade@3.2.0: {} + + escape-string-regexp@4.0.0: {} + + eslint-plugin-react-hooks@7.0.1(eslint@9.39.4): + dependencies: + '@babel/core': 7.29.0 + '@babel/parser': 7.29.2 + eslint: 9.39.4 + hermes-parser: 0.25.1 + zod: 4.3.6 + zod-validation-error: 4.0.2(zod@4.3.6) + transitivePeerDependencies: + - supports-color + + eslint-plugin-react-refresh@0.5.2(eslint@9.39.4): + dependencies: + eslint: 9.39.4 + + eslint-scope@8.4.0: + dependencies: + esrecurse: 4.3.0 + estraverse: 5.3.0 + + eslint-visitor-keys@3.4.3: {} + + eslint-visitor-keys@4.2.1: {} + + eslint-visitor-keys@5.0.1: {} + + eslint@9.39.4: + dependencies: + '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.4) + '@eslint-community/regexpp': 4.12.2 + '@eslint/config-array': 0.21.2 + '@eslint/config-helpers': 0.4.2 + '@eslint/core': 0.17.0 + '@eslint/eslintrc': 3.3.5 + '@eslint/js': 9.39.4 + '@eslint/plugin-kit': 0.4.1 + '@humanfs/node': 0.16.7 + '@humanwhocodes/module-importer': 1.0.1 + '@humanwhocodes/retry': 0.4.3 + '@types/estree': 1.0.8 + ajv: 6.14.0 + chalk: 4.1.2 + cross-spawn: 7.0.6 + debug: 4.4.3 + escape-string-regexp: 4.0.0 + eslint-scope: 8.4.0 + eslint-visitor-keys: 4.2.1 + espree: 10.4.0 + esquery: 1.7.0 + esutils: 2.0.3 + fast-deep-equal: 3.1.3 + file-entry-cache: 8.0.0 + find-up: 5.0.0 + glob-parent: 6.0.2 + ignore: 5.3.2 + imurmurhash: 0.1.4 + is-glob: 4.0.3 + json-stable-stringify-without-jsonify: 1.0.1 + lodash.merge: 4.6.2 + minimatch: 3.1.5 + natural-compare: 1.4.0 + optionator: 0.9.4 + transitivePeerDependencies: + - supports-color + + espree@10.4.0: + dependencies: + acorn: 8.16.0 + acorn-jsx: 5.3.2(acorn@8.16.0) + eslint-visitor-keys: 4.2.1 + + esquery@1.7.0: + dependencies: + estraverse: 5.3.0 + + esrecurse@4.3.0: + dependencies: + estraverse: 5.3.0 + + estraverse@5.3.0: {} + + esutils@2.0.3: {} + + fast-deep-equal@3.1.3: {} + + fast-json-stable-stringify@2.1.0: {} + + fast-levenshtein@2.0.6: {} + + fdir@6.5.0(picomatch@4.0.4): + optionalDependencies: + picomatch: 4.0.4 + + file-entry-cache@8.0.0: + dependencies: + flat-cache: 4.0.1 + + find-up@5.0.0: + dependencies: + locate-path: 6.0.0 + path-exists: 4.0.0 + + flat-cache@4.0.1: + dependencies: + flatted: 3.4.2 + keyv: 4.5.4 + + flatted@3.4.2: {} + + fsevents@2.3.3: + optional: true + + gensync@1.0.0-beta.2: {} + + glob-parent@6.0.2: + dependencies: + is-glob: 4.0.3 + + globals@14.0.0: {} + + globals@17.5.0: {} + + has-flag@4.0.0: {} + + hermes-estree@0.25.1: {} + + hermes-parser@0.25.1: + dependencies: + hermes-estree: 0.25.1 + + ignore@5.3.2: {} + + ignore@7.0.5: {} + + import-fresh@3.3.1: + dependencies: + parent-module: 1.0.1 + resolve-from: 4.0.0 + + imurmurhash@0.1.4: {} + + is-extglob@2.1.1: {} + + is-glob@4.0.3: + dependencies: + is-extglob: 2.1.1 + + isexe@2.0.0: {} + + js-tokens@4.0.0: {} + + js-yaml@4.1.1: + dependencies: + argparse: 2.0.1 + + jsesc@3.1.0: {} + + json-buffer@3.0.1: {} + + json-schema-traverse@0.4.1: {} + + json-stable-stringify-without-jsonify@1.0.1: {} + + json5@2.2.3: {} + + keyv@4.5.4: + dependencies: + json-buffer: 3.0.1 + + levn@0.4.1: + dependencies: + prelude-ls: 1.2.1 + type-check: 0.4.0 + + lightningcss-android-arm64@1.32.0: + optional: true + + lightningcss-darwin-arm64@1.32.0: + optional: true + + lightningcss-darwin-x64@1.32.0: + optional: true + + lightningcss-freebsd-x64@1.32.0: + optional: true + + lightningcss-linux-arm-gnueabihf@1.32.0: + optional: true + + lightningcss-linux-arm64-gnu@1.32.0: + optional: true + + lightningcss-linux-arm64-musl@1.32.0: + optional: true + + lightningcss-linux-x64-gnu@1.32.0: + optional: true + + lightningcss-linux-x64-musl@1.32.0: + optional: true + + lightningcss-win32-arm64-msvc@1.32.0: + optional: true + + lightningcss-win32-x64-msvc@1.32.0: + optional: true + + lightningcss@1.32.0: + dependencies: + detect-libc: 2.1.2 + optionalDependencies: + lightningcss-android-arm64: 1.32.0 + lightningcss-darwin-arm64: 1.32.0 + lightningcss-darwin-x64: 1.32.0 + lightningcss-freebsd-x64: 1.32.0 + lightningcss-linux-arm-gnueabihf: 1.32.0 + lightningcss-linux-arm64-gnu: 1.32.0 + lightningcss-linux-arm64-musl: 1.32.0 + lightningcss-linux-x64-gnu: 1.32.0 + lightningcss-linux-x64-musl: 1.32.0 + lightningcss-win32-arm64-msvc: 1.32.0 + lightningcss-win32-x64-msvc: 1.32.0 + + locate-path@6.0.0: + dependencies: + p-locate: 5.0.0 + + lodash.merge@4.6.2: {} + + lru-cache@5.1.1: + dependencies: + yallist: 3.1.1 + + minimatch@10.2.5: + dependencies: + brace-expansion: 5.0.5 + + minimatch@3.1.5: + dependencies: + brace-expansion: 1.1.14 + + ms@2.1.3: {} + + nanoid@3.3.11: {} + + natural-compare@1.4.0: {} + + node-releases@2.0.37: {} + + optionator@0.9.4: + dependencies: + deep-is: 0.1.4 + fast-levenshtein: 2.0.6 + levn: 0.4.1 + prelude-ls: 1.2.1 + type-check: 0.4.0 + word-wrap: 1.2.5 + + p-limit@3.1.0: + dependencies: + yocto-queue: 0.1.0 + + p-locate@5.0.0: + dependencies: + p-limit: 3.1.0 + + parent-module@1.0.1: + dependencies: + callsites: 3.1.0 + + path-exists@4.0.0: {} + + path-key@3.1.1: {} + + picocolors@1.1.1: {} + + picomatch@4.0.4: {} + + postcss@8.5.10: + dependencies: + nanoid: 3.3.11 + picocolors: 1.1.1 + source-map-js: 1.2.1 + + prelude-ls@1.2.1: {} + + punycode@2.3.1: {} + + react-dom@19.2.5(react@19.2.5): + dependencies: + react: 19.2.5 + scheduler: 0.27.0 + + react@19.2.5: {} + + resolve-from@4.0.0: {} + + rolldown@1.0.0-rc.15: + dependencies: + '@oxc-project/types': 0.124.0 + '@rolldown/pluginutils': 1.0.0-rc.15 + optionalDependencies: + '@rolldown/binding-android-arm64': 1.0.0-rc.15 + '@rolldown/binding-darwin-arm64': 1.0.0-rc.15 + '@rolldown/binding-darwin-x64': 1.0.0-rc.15 + '@rolldown/binding-freebsd-x64': 1.0.0-rc.15 + '@rolldown/binding-linux-arm-gnueabihf': 1.0.0-rc.15 + '@rolldown/binding-linux-arm64-gnu': 1.0.0-rc.15 + '@rolldown/binding-linux-arm64-musl': 1.0.0-rc.15 + '@rolldown/binding-linux-ppc64-gnu': 1.0.0-rc.15 + '@rolldown/binding-linux-s390x-gnu': 1.0.0-rc.15 + '@rolldown/binding-linux-x64-gnu': 1.0.0-rc.15 + '@rolldown/binding-linux-x64-musl': 1.0.0-rc.15 + '@rolldown/binding-openharmony-arm64': 1.0.0-rc.15 + '@rolldown/binding-wasm32-wasi': 1.0.0-rc.15 + '@rolldown/binding-win32-arm64-msvc': 1.0.0-rc.15 + '@rolldown/binding-win32-x64-msvc': 1.0.0-rc.15 + + scheduler@0.27.0: {} + + semver@6.3.1: {} + + semver@7.7.4: {} + + shebang-command@2.0.0: + dependencies: + shebang-regex: 3.0.0 + + shebang-regex@3.0.0: {} + + source-map-js@1.2.1: {} + + strip-json-comments@3.1.1: {} + + supports-color@7.2.0: + dependencies: + has-flag: 4.0.0 + + tinyglobby@0.2.16: + dependencies: + fdir: 6.5.0(picomatch@4.0.4) + picomatch: 4.0.4 + + ts-api-utils@2.5.0(typescript@6.0.2): + dependencies: + typescript: 6.0.2 + + tslib@2.8.1: + optional: true + + type-check@0.4.0: + dependencies: + prelude-ls: 1.2.1 + + typescript-eslint@8.58.2(eslint@9.39.4)(typescript@6.0.2): + dependencies: + '@typescript-eslint/eslint-plugin': 8.58.2(@typescript-eslint/parser@8.58.2(eslint@9.39.4)(typescript@6.0.2))(eslint@9.39.4)(typescript@6.0.2) + '@typescript-eslint/parser': 8.58.2(eslint@9.39.4)(typescript@6.0.2) + '@typescript-eslint/typescript-estree': 8.58.2(typescript@6.0.2) + '@typescript-eslint/utils': 8.58.2(eslint@9.39.4)(typescript@6.0.2) + eslint: 9.39.4 + typescript: 6.0.2 + transitivePeerDependencies: + - supports-color + + typescript@6.0.2: {} + + undici-types@7.16.0: {} + + update-browserslist-db@1.2.3(browserslist@4.28.2): + dependencies: + browserslist: 4.28.2 + escalade: 3.2.0 + picocolors: 1.1.1 + + uri-js@4.4.1: + dependencies: + punycode: 2.3.1 + + vite@8.0.8(@types/node@24.12.2): + dependencies: + lightningcss: 1.32.0 + picomatch: 4.0.4 + postcss: 8.5.10 + rolldown: 1.0.0-rc.15 + tinyglobby: 0.2.16 + optionalDependencies: + '@types/node': 24.12.2 + fsevents: 2.3.3 + + which@2.0.2: + dependencies: + isexe: 2.0.0 + + word-wrap@1.2.5: {} + + yallist@3.1.1: {} + + yocto-queue@0.1.0: {} + + zod-validation-error@4.0.2(zod@4.3.6): + dependencies: + zod: 4.3.6 + + zod@4.3.6: {} diff --git a/frontend/src/App.css b/frontend/src/App.css new file mode 100644 index 0000000000000000000000000000000000000000..91f1497260cbbf201a85accc172365aeb897c00f --- /dev/null +++ b/frontend/src/App.css @@ -0,0 +1,262 @@ +* { + box-sizing: border-box; + margin: 0; + padding: 0; +} + +body { + font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; + background: #0f1117; + color: #e0e0e0; +} + +.app-layout { + display: flex; + height: 100vh; +} + +/* ── Sidebar ──────────────────────────────────────────────────────────── */ + +.sidebar { + width: 320px; + background: #1a1d27; + padding: 20px; + display: flex; + flex-direction: column; + gap: 16px; + overflow-y: auto; + border-right: 1px solid #2a2d37; +} + +.app-title { + font-size: 20px; + font-weight: 600; + color: #fff; +} + +.sidebar-section { + display: flex; + flex-direction: column; + gap: 8px; +} + +.toggle-label { + display: flex; + align-items: center; + gap: 8px; + cursor: pointer; + font-size: 14px; +} + +/* ── Forms ─────────────────────────────────────────────────────────── */ + +.persona-selector { + display: flex; + flex-direction: column; + gap: 4px; +} + +label { + font-size: 12px; + color: #888; + text-transform: uppercase; + letter-spacing: 0.5px; +} + +select, input[type="text"] { + background: #252830; + color: #e0e0e0; + border: 1px solid #3a3d47; + border-radius: 6px; + padding: 8px 10px; + font-size: 14px; + outline: none; +} + +select:focus, input[type="text"]:focus { + border-color: #5b8def; +} + +/* ── Webcam ────────────────────────────────────────────────────────── */ + +.webcam-container { + border-radius: 8px; + overflow: hidden; + background: #252830; +} + +.webcam-placeholder, .webcam-error { + padding: 24px; + text-align: center; + color: #666; + font-size: 13px; +} + +.webcam-error { + color: #e55; +} + +/* ── Sensing status ───────────────────────────────────────────────── */ + +.sensing-off { + color: #666; + font-size: 13px; +} + +.sensing-status { + display: flex; + flex-direction: column; + gap: 4px; +} + +.sensing-row { + display: flex; + justify-content: space-between; + font-size: 13px; +} + +.sensing-label { + color: #888; +} + +.sensing-value { + color: #ccc; + font-weight: 500; +} + +/* ── Latency metrics ──────────────────────────────────────────────── */ + +.latency-metrics h3 { + font-size: 12px; + color: #888; + text-transform: uppercase; + letter-spacing: 0.5px; + margin-bottom: 6px; +} + +.metric-row { + display: flex; + justify-content: space-between; + font-size: 13px; + padding: 2px 0; +} + +.metric-label { + color: #888; +} + +.metric-value { + color: #ccc; + font-family: monospace; +} + +.no-metrics { + color: #555; + font-size: 13px; +} + +/* ── Main content ─────────────────────────────────────────────────── */ + +.main-content { + flex: 1; + display: flex; + flex-direction: column; +} + +/* ── Chat panel ───────────────────────────────────────────────────── */ + +.chat-panel { + display: flex; + flex-direction: column; + height: 100%; +} + +.chat-header { + padding: 16px 24px; + font-size: 16px; + font-weight: 600; + border-bottom: 1px solid #2a2d37; + background: #1a1d27; +} + +.chat-messages { + flex: 1; + overflow-y: auto; + padding: 20px 24px; + display: flex; + flex-direction: column; + gap: 12px; +} + +.chat-bubble { + max-width: 75%; + padding: 10px 14px; + border-radius: 12px; + font-size: 14px; + line-height: 1.5; +} + +.chat-bubble.partner { + align-self: flex-end; + background: #2a4a8a; + border-bottom-right-radius: 4px; +} + +.chat-bubble.aac_user { + align-self: flex-start; + background: #252830; + border-bottom-left-radius: 4px; +} + +.chat-bubble.loading { + opacity: 0.6; +} + +.chat-role { + display: block; + font-size: 11px; + color: #888; + margin-bottom: 4px; + text-transform: uppercase; + letter-spacing: 0.5px; +} + +.chat-bubble p { + margin: 0; +} + +.chat-input-row { + display: flex; + gap: 8px; + padding: 16px 24px; + border-top: 1px solid #2a2d37; + background: #1a1d27; +} + +.chat-input-row input { + flex: 1; +} + +.chat-input-row button { + background: #5b8def; + color: #fff; + border: none; + border-radius: 6px; + padding: 8px 20px; + font-size: 14px; + cursor: pointer; +} + +.chat-input-row button:disabled { + opacity: 0.4; + cursor: not-allowed; +} + +.chat-input-row button:hover:not(:disabled) { + background: #4a7cde; +} + +.error { + color: #e55; + font-size: 13px; +} diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx new file mode 100644 index 0000000000000000000000000000000000000000..81290784c7d862391448c72fd6f0d0c014604700 --- /dev/null +++ b/frontend/src/App.tsx @@ -0,0 +1,133 @@ +import { useState, useCallback, useEffect, useRef } from "react"; +import type { Persona, Affect, ChatMessage, LatencyLog } from "./types"; +import { resetSession, checkHealth } from "./lib/api"; +import { useWebcam } from "./hooks/useWebcam"; +import { useSensing } from "./hooks/useSensing"; +import { PersonaSelector } from "./components/PersonaSelector"; +import { ChatPanel } from "./components/ChatPanel"; +import { WebcamSensing } from "./components/WebcamSensing"; +import { SensingStatus } from "./components/SensingStatus"; +import { LatencyMetrics } from "./components/LatencyMetrics"; +import "./App.css"; + +function App() { + const [persona, setPersona] = useState(null); + const [messages, setMessages] = useState([]); + const [latency, setLatency] = useState(null); + const [webcamEnabled, setWebcamEnabled] = useState(false); + const [affectOverride, setAffectOverride] = useState(null); + const [backendReady, setBackendReady] = useState(false); + const healthPoll = useRef>(undefined); + + useEffect(() => { + async function poll() { + const ready = await checkHealth(); + if (ready) { + setBackendReady(true); + clearInterval(healthPoll.current); + } + } + poll(); + healthPoll.current = setInterval(poll, 2000); + return () => clearInterval(healthPoll.current); + }, []); + + const { sensing, ready, initError, init, processFrame, clearAirWrittenText, resetCalibration } = + useSensing(); + + const onFrame = useCallback( + (video: HTMLVideoElement, timestamp: number) => { + processFrame(video, timestamp); + }, + [processFrame] + ); + + const { videoRef, active, error } = useWebcam({ + enabled: webcamEnabled && ready, + onFrame, + }); + + async function handleWebcamToggle() { + if (!webcamEnabled) { + const ok = await init(); + if (ok) setWebcamEnabled(true); + } else { + setWebcamEnabled(false); + resetCalibration(); + } + } + + async function handlePersonaSelect(p: Persona) { + setPersona(p); + setMessages([]); + setLatency(null); + try { + await resetSession(p.id); + } catch { + // Session reset failed — non-critical, continue with fresh UI state + } + } + + return ( +
+ + +
+ +
+
+ ); +} + +export default App; diff --git a/frontend/src/components/ChatPanel.tsx b/frontend/src/components/ChatPanel.tsx new file mode 100644 index 0000000000000000000000000000000000000000..357683605c603f2cefe45a1a1f7a891b5bb20973 --- /dev/null +++ b/frontend/src/components/ChatPanel.tsx @@ -0,0 +1,116 @@ +import { useState, useRef, useEffect } from "react"; +import type { ChatMessage, SensingState, Affect, LatencyLog } from "../types"; +import { sendChat } from "../lib/api"; + +interface Props { + userId: string | null; + personaName: string; + sensing: SensingState; + affectOverride: Affect | null; + onAirTextConsumed: () => void; + messages: ChatMessage[]; + setMessages: React.Dispatch>; + onLatency: (latency: LatencyLog) => void; + backendReady: boolean; +} + +export function ChatPanel({ + userId, + personaName, + sensing, + affectOverride, + onAirTextConsumed, + messages, + setMessages, + onLatency, + backendReady, +}: Props) { + const [input, setInput] = useState(""); + const [loading, setLoading] = useState(false); + const bottomRef = useRef(null); + + useEffect(() => { + bottomRef.current?.scrollIntoView({ behavior: "smooth" }); + }, [messages]); + + async function handleSend() { + if (!input.trim() || !userId || !backendReady || loading) return; + + const query = input.trim(); + setInput(""); + setMessages((prev) => [...prev, { role: "partner", content: query }]); + setLoading(true); + + const airText = sensing.airWrittenText || null; + try { + const res = await sendChat({ + user_id: userId, + query, + affect_override: affectOverride ?? sensing.affect, + gesture_tag: sensing.gestureTag, + gaze_bucket: sensing.gazeBucket, + air_written_text: airText, + }); + + setMessages((prev) => [ + ...prev, + { + role: "aac_user", + content: res.response, + latency: res.latency, + affect: res.affect, + }, + ]); + onLatency(res.latency); + } catch (e) { + setMessages((prev) => [ + ...prev, + { + role: "aac_user", + content: `Error: ${e instanceof Error ? e.message : "request failed"}`, + }, + ]); + } finally { + if (airText) onAirTextConsumed(); + setLoading(false); + } + } + + return ( +
+
+ Talking as: {personaName || "select a persona"} +
+
+ {messages.map((msg, i) => ( +
+ + {msg.role === "partner" ? "Partner" : "AAC User"} + +

{msg.content}

+
+ ))} + {loading && ( +
+ AAC User +

Generating...

+
+ )} +
+
+
+ setInput(e.target.value)} + onKeyDown={(e) => e.key === "Enter" && !e.shiftKey && handleSend()} + placeholder={backendReady ? "Type as the communication partner..." : "Waiting for backend to load models..."} + disabled={!userId || loading || !backendReady} + /> + +
+
+ ); +} diff --git a/frontend/src/components/LatencyMetrics.tsx b/frontend/src/components/LatencyMetrics.tsx new file mode 100644 index 0000000000000000000000000000000000000000..41fb4080a6f102c68e5fdda6859f731db6c5c408 --- /dev/null +++ b/frontend/src/components/LatencyMetrics.tsx @@ -0,0 +1,29 @@ +import type { LatencyLog } from "../types"; + +interface Props { + latency: LatencyLog | null; +} + +const FIELDS: { key: keyof LatencyLog; label: string }[] = [ + { key: "t_sensing", label: "Sensing" }, + { key: "t_intent", label: "Intent" }, + { key: "t_retrieval", label: "Retrieval" }, + { key: "t_generation", label: "Generation" }, + { key: "t_total", label: "Total" }, +]; + +export function LatencyMetrics({ latency }: Props) { + if (!latency) return

No turn yet

; + + return ( +
+

Latency

+ {FIELDS.map(({ key, label }) => ( +
+ {label} + {(latency[key] ?? 0).toFixed(3)}s +
+ ))} +
+ ); +} diff --git a/frontend/src/components/PersonaSelector.tsx b/frontend/src/components/PersonaSelector.tsx new file mode 100644 index 0000000000000000000000000000000000000000..4f38caaa96d58329e6c0d448b193d58048c79d27 --- /dev/null +++ b/frontend/src/components/PersonaSelector.tsx @@ -0,0 +1,42 @@ +import { useEffect, useState } from "react"; +import type { Persona } from "../types"; +import { fetchUsers } from "../lib/api"; + +interface Props { + selected: string | null; + onSelect: (persona: Persona) => void; +} + +export function PersonaSelector({ selected, onSelect }: Props) { + const [personas, setPersonas] = useState([]); + const [error, setError] = useState(null); + + useEffect(() => { + fetchUsers() + .then(setPersonas) + .catch(() => setError("Cannot reach API — start the FastAPI server")); + }, []); + + if (error) return

{error}

; + + return ( +
+ + +
+ ); +} diff --git a/frontend/src/components/SensingStatus.tsx b/frontend/src/components/SensingStatus.tsx new file mode 100644 index 0000000000000000000000000000000000000000..809f50a1e9473c2f998847e00bfc7049aaba6714 --- /dev/null +++ b/frontend/src/components/SensingStatus.tsx @@ -0,0 +1,49 @@ +import type { SensingState } from "../types"; + +const AFFECT_EMOJI: Record = { + HAPPY: "\ud83d\ude0a", + FRUSTRATED: "\ud83d\ude24", + NEUTRAL: "\ud83d\ude10", + SURPRISED: "\ud83d\ude32", +}; + +interface Props { + sensing: SensingState; + webcamActive: boolean; +} + +export function SensingStatus({ sensing, webcamActive }: Props) { + if (!webcamActive) { + return

Webcam off

; + } + + return ( +
+
+ Affect + + {AFFECT_EMOJI[sensing.affect ?? "NEUTRAL"]}{" "} + {sensing.affect ?? "NEUTRAL"} + +
+
+ Gesture + + {sensing.gestureTag ?? "none"} + +
+
+ Gaze + + {sensing.gazeBucket ?? "none"} + +
+ {sensing.airWrittenText && ( +
+ Air-written + {sensing.airWrittenText} +
+ )} +
+ ); +} diff --git a/frontend/src/components/WebcamSensing.tsx b/frontend/src/components/WebcamSensing.tsx new file mode 100644 index 0000000000000000000000000000000000000000..d33a742591c804956619da7c050273c570075a68 --- /dev/null +++ b/frontend/src/components/WebcamSensing.tsx @@ -0,0 +1,30 @@ +import type { RefObject } from "react"; + +interface Props { + videoRef: RefObject; + active: boolean; + error: string | null; +} + +export function WebcamSensing({ videoRef, active, error }: Props) { + return ( +
+
+ ); +} diff --git a/frontend/src/hooks/useSensing.ts b/frontend/src/hooks/useSensing.ts new file mode 100644 index 0000000000000000000000000000000000000000..1df43b1f588d8e1a791a363e8d4e07a54762e310 --- /dev/null +++ b/frontend/src/hooks/useSensing.ts @@ -0,0 +1,164 @@ +import { useRef, useCallback, useState, useEffect } from "react"; +import { + FaceLandmarker, + HandLandmarker, + FilesetResolver, +} from "@mediapipe/tasks-vision"; +import type { SensingState } from "../types"; +import { + computeAffectVector, + classifyAffect, + classifyGesture, + GazeTracker, + AirWriter, +} from "../lib/sensing"; + +const EMA_ALPHA = 0.3; + +export function useSensing() { + const faceLandmarkerRef = useRef(null); + const handLandmarkerRef = useRef(null); + const gazeTrackerRef = useRef(new GazeTracker()); + const airWriterRef = useRef(new AirWriter()); + const neutralLCPRef = useRef(null); + const smoothedRef = useRef({ MAR: 0, EAR: 0.3, BRI: -0.3, LCP: 0 }); + const initingRef = useRef(false); + const [ready, setReady] = useState(false); + const [initError, setInitError] = useState(null); + const [sensing, setSensing] = useState({ + affect: null, + gestureTag: null, + gazeBucket: null, + airWrittenText: "", + }); + + // Cleanup MediaPipe resources on unmount + useEffect(() => { + return () => { + faceLandmarkerRef.current?.close(); + handLandmarkerRef.current?.close(); + faceLandmarkerRef.current = null; + handLandmarkerRef.current = null; + }; + }, []); + + const init = useCallback(async (): Promise => { + if (faceLandmarkerRef.current || initingRef.current) return true; + initingRef.current = true; + try { + const vision = await FilesetResolver.forVisionTasks( + "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@latest/wasm" + ); + faceLandmarkerRef.current = await FaceLandmarker.createFromOptions( + vision, + { + baseOptions: { + modelAssetPath: + "https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task", + delegate: "GPU", + }, + runningMode: "VIDEO", + numFaces: 1, + outputFaceBlendshapes: false, + outputFacialTransformationMatrixes: false, + } + ); + handLandmarkerRef.current = await HandLandmarker.createFromOptions( + vision, + { + baseOptions: { + modelAssetPath: + "https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task", + delegate: "GPU", + }, + runningMode: "VIDEO", + numHands: 1, + } + ); + setReady(true); + return true; + } catch (e) { + setInitError( + e instanceof Error ? e.message : "Failed to load MediaPipe models" + ); + return false; + } finally { + initingRef.current = false; + } + }, []); + + const processFrame = useCallback( + (video: HTMLVideoElement, timestamp: number) => { + const faceLandmarker = faceLandmarkerRef.current; + const handLandmarker = handLandmarkerRef.current; + if (!faceLandmarker || !handLandmarker) return; + + let affect: SensingState["affect"] = null; + let gazeBucket: SensingState["gazeBucket"] = null; + + const faceResult = faceLandmarker.detectForVideo(video, timestamp); + if (faceResult.faceLandmarks && faceResult.faceLandmarks.length > 0) { + const landmarks = faceResult.faceLandmarks[0]; + + if (neutralLCPRef.current === null) { + neutralLCPRef.current = + (landmarks[61].x + landmarks[291].x) / 2; + } + + const raw = computeAffectVector(landmarks, neutralLCPRef.current); + + const prev = smoothedRef.current; + const smoothed = { + MAR: EMA_ALPHA * raw.MAR + (1 - EMA_ALPHA) * prev.MAR, + EAR: EMA_ALPHA * raw.EAR + (1 - EMA_ALPHA) * prev.EAR, + BRI: EMA_ALPHA * raw.BRI + (1 - EMA_ALPHA) * prev.BRI, + LCP: EMA_ALPHA * raw.LCP + (1 - EMA_ALPHA) * prev.LCP, + }; + smoothedRef.current = smoothed; + + affect = classifyAffect(smoothed); + gazeBucket = gazeTrackerRef.current.process(landmarks); + } + + let gestureTag: SensingState["gestureTag"] = null; + + const handResult = handLandmarker.detectForVideo(video, timestamp); + if (handResult.landmarks && handResult.landmarks.length > 0) { + const handLandmarks = handResult.landmarks[0]; + gestureTag = classifyGesture(handLandmarks); + airWriterRef.current.processHandLandmarks( + handLandmarks, + video.videoWidth, + video.videoHeight + ); + } else { + airWriterRef.current.noHand(); + } + + const newAirText = airWriterRef.current.getText(); + + setSensing((prev) => ({ + affect: affect ?? prev.affect, + gestureTag: gestureTag ?? prev.gestureTag, + gazeBucket: gazeBucket ?? prev.gazeBucket, + airWrittenText: newAirText + ? prev.airWrittenText + newAirText + : prev.airWrittenText, + })); + }, + [] + ); + + const clearAirWrittenText = useCallback(() => { + setSensing((prev) => ({ ...prev, airWrittenText: "" })); + }, []); + + const resetCalibration = useCallback(() => { + neutralLCPRef.current = null; + smoothedRef.current = { MAR: 0, EAR: 0.3, BRI: -0.3, LCP: 0 }; + gazeTrackerRef.current.reset(); + setSensing({ affect: null, gestureTag: null, gazeBucket: null, airWrittenText: "" }); + }, []); + + return { sensing, ready, initError, init, processFrame, clearAirWrittenText, resetCalibration }; +} diff --git a/frontend/src/hooks/useWebcam.ts b/frontend/src/hooks/useWebcam.ts new file mode 100644 index 0000000000000000000000000000000000000000..5294f696812b81560224a3dabe69f29fe56b2390 --- /dev/null +++ b/frontend/src/hooks/useWebcam.ts @@ -0,0 +1,102 @@ +import { useRef, useEffect, useState } from "react"; + +interface UseWebcamOptions { + enabled: boolean; + onFrame?: (video: HTMLVideoElement, timestamp: number) => void; + processEveryN?: number; +} + +export function useWebcam({ + enabled, + onFrame, + processEveryN = 3, +}: UseWebcamOptions) { + const videoRef = useRef(null); + const streamRef = useRef(null); + const frameCount = useRef(0); + const rafId = useRef(0); + const onFrameRef = useRef(onFrame); + const [error, setError] = useState(null); + const [active, setActive] = useState(false); + + useEffect(() => { + onFrameRef.current = onFrame; + }, [onFrame]); + + function teardown() { + if (rafId.current) cancelAnimationFrame(rafId.current); + rafId.current = 0; + if (streamRef.current) { + streamRef.current.getTracks().forEach((t) => t.stop()); + streamRef.current = null; + } + if (videoRef.current) { + videoRef.current.srcObject = null; + } + } + + useEffect(() => { + if (!enabled) { + teardown(); + setActive(false); + return; + } + + let cancelled = false; + + async function start() { + try { + const stream = await navigator.mediaDevices.getUserMedia({ + video: { facingMode: "user", width: 640, height: 480 }, + }); + if (cancelled) { + stream.getTracks().forEach((t) => t.stop()); + return; + } + streamRef.current = stream; + if (videoRef.current) { + videoRef.current.srcObject = stream; + await videoRef.current.play(); + } + if (cancelled) { + teardown(); + return; + } + setActive(true); + setError(null); + + function loop(timestamp: number) { + if (cancelled) return; + frameCount.current++; + if ( + frameCount.current % processEveryN === 0 && + videoRef.current && + onFrameRef.current + ) { + onFrameRef.current(videoRef.current, timestamp); + } + rafId.current = requestAnimationFrame(loop); + } + rafId.current = requestAnimationFrame(loop); + } catch (e) { + if (!cancelled) { + setError( + e instanceof Error ? e.message : "Webcam access denied" + ); + setActive(false); + } + } + } + + start(); + + return () => { + cancelled = true; + teardown(); + setActive(false); + }; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [enabled, processEveryN]); + + return { videoRef, active, error }; +} diff --git a/frontend/src/index.css b/frontend/src/index.css new file mode 100644 index 0000000000000000000000000000000000000000..5fb33130220482ba51683f2f43660cca10f174d9 --- /dev/null +++ b/frontend/src/index.css @@ -0,0 +1,111 @@ +:root { + --text: #6b6375; + --text-h: #08060d; + --bg: #fff; + --border: #e5e4e7; + --code-bg: #f4f3ec; + --accent: #aa3bff; + --accent-bg: rgba(170, 59, 255, 0.1); + --accent-border: rgba(170, 59, 255, 0.5); + --social-bg: rgba(244, 243, 236, 0.5); + --shadow: + rgba(0, 0, 0, 0.1) 0 10px 15px -3px, rgba(0, 0, 0, 0.05) 0 4px 6px -2px; + + --sans: system-ui, 'Segoe UI', Roboto, sans-serif; + --heading: system-ui, 'Segoe UI', Roboto, sans-serif; + --mono: ui-monospace, Consolas, monospace; + + font: 18px/145% var(--sans); + letter-spacing: 0.18px; + color-scheme: light dark; + color: var(--text); + background: var(--bg); + font-synthesis: none; + text-rendering: optimizeLegibility; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; + + @media (max-width: 1024px) { + font-size: 16px; + } +} + +@media (prefers-color-scheme: dark) { + :root { + --text: #9ca3af; + --text-h: #f3f4f6; + --bg: #16171d; + --border: #2e303a; + --code-bg: #1f2028; + --accent: #c084fc; + --accent-bg: rgba(192, 132, 252, 0.15); + --accent-border: rgba(192, 132, 252, 0.5); + --social-bg: rgba(47, 48, 58, 0.5); + --shadow: + rgba(0, 0, 0, 0.4) 0 10px 15px -3px, rgba(0, 0, 0, 0.25) 0 4px 6px -2px; + } + + #social .button-icon { + filter: invert(1) brightness(2); + } +} + +#root { + width: 1126px; + max-width: 100%; + margin: 0 auto; + text-align: center; + border-inline: 1px solid var(--border); + min-height: 100svh; + display: flex; + flex-direction: column; + box-sizing: border-box; +} + +body { + margin: 0; +} + +h1, +h2 { + font-family: var(--heading); + font-weight: 500; + color: var(--text-h); +} + +h1 { + font-size: 56px; + letter-spacing: -1.68px; + margin: 32px 0; + @media (max-width: 1024px) { + font-size: 36px; + margin: 20px 0; + } +} +h2 { + font-size: 24px; + line-height: 118%; + letter-spacing: -0.24px; + margin: 0 0 8px; + @media (max-width: 1024px) { + font-size: 20px; + } +} +p { + margin: 0; +} + +code, +.counter { + font-family: var(--mono); + display: inline-flex; + border-radius: 4px; + color: var(--text-h); +} + +code { + font-size: 15px; + line-height: 135%; + padding: 4px 8px; + background: var(--code-bg); +} diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts new file mode 100644 index 0000000000000000000000000000000000000000..0ec2b9c35db92fcd8c7d454621aec8d01ed004c3 --- /dev/null +++ b/frontend/src/lib/api.ts @@ -0,0 +1,39 @@ +import type { ChatRequest, ChatResponse, Persona } from "../types"; + +const API_BASE = ""; + +export async function fetchUsers(): Promise { + const res = await fetch(`${API_BASE}/users`); + if (!res.ok) throw new Error(`API error: ${res.status}`); + const data = await res.json(); + return data.users; +} + +export async function sendChat(req: ChatRequest): Promise { + const res = await fetch(`${API_BASE}/chat`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(req), + }); + if (!res.ok) throw new Error(`API error: ${res.status}`); + return res.json(); +} + +export async function resetSession(userId: string): Promise { + const res = await fetch( + `${API_BASE}/session/reset?user_id=${encodeURIComponent(userId)}`, + { method: "POST" } + ); + if (!res.ok) throw new Error(`API error: ${res.status}`); +} + +export async function checkHealth(): Promise { + try { + const res = await fetch(`${API_BASE}/health`); + if (!res.ok) return false; + const data = await res.json(); + return data.models_ready === true; + } catch { + return false; + } +} diff --git a/frontend/src/lib/sensing.ts b/frontend/src/lib/sensing.ts new file mode 100644 index 0000000000000000000000000000000000000000..a7c6bd4dd5a592a176e8f501715f8c9b923b000f --- /dev/null +++ b/frontend/src/lib/sensing.ts @@ -0,0 +1,330 @@ +import type { Affect, GestureName, MemoryBucket } from "../types"; + +// ── Affect classification (ported from backend/sensing/face_mesh.py) ──────── + +interface AffectVector { + MAR: number; + EAR: number; + BRI: number; + LCP: number; +} + +export function classifyAffect(v: AffectVector): Affect { + // BRI is relative (browMid.y - eyeCenter.y) / interOcular — more negative = brows raised higher + // LCP is relative to calibrated neutral — positive = corners pulled up (smile) + // MAR is absolute ratio — higher = mouth more open + // EAR is absolute ratio — lower = eyes more closed + if (v.BRI < -0.35 && v.MAR > 0.4) return "SURPRISED"; + if (v.EAR < 0.12 && v.LCP < -0.005) return "FRUSTRATED"; + if (v.LCP > 0.005) return "HAPPY"; + return "NEUTRAL"; +} + +// Face landmark indices (MediaPipe 478-point mesh) +const MOUTH_TOP = 13, MOUTH_BOTTOM = 14, MOUTH_LEFT = 61, MOUTH_RIGHT = 291; +const EYE_TOP = 159, EYE_BOTTOM = 145, EYE_LEFT = 33, EYE_RIGHT = 133; +const BROW_LEFT = 70, BROW_RIGHT = 300; +const CORNER_LEFT = 61, CORNER_RIGHT = 291; + +function dist(a: { x: number; y: number }, b: { x: number; y: number }): number { + return Math.sqrt((a.x - b.x) ** 2 + (a.y - b.y) ** 2); +} + +export function computeAffectVector( + landmarks: { x: number; y: number }[], + neutralLCP: number +): AffectVector { + const MAR = + dist(landmarks[MOUTH_TOP], landmarks[MOUTH_BOTTOM]) / + (dist(landmarks[MOUTH_LEFT], landmarks[MOUTH_RIGHT]) + 1e-6); + + const EAR = + dist(landmarks[EYE_TOP], landmarks[EYE_BOTTOM]) / + (dist(landmarks[EYE_LEFT], landmarks[EYE_RIGHT]) + 1e-6); + + const eyeCenter = { + x: (landmarks[EYE_LEFT].x + landmarks[EYE_RIGHT].x) / 2, + y: (landmarks[EYE_LEFT].y + landmarks[EYE_RIGHT].y) / 2, + }; + const interOcular = dist(landmarks[EYE_LEFT], landmarks[EYE_RIGHT]); + const browMid = { + x: (landmarks[BROW_LEFT].x + landmarks[BROW_RIGHT].x) / 2, + y: (landmarks[BROW_LEFT].y + landmarks[BROW_RIGHT].y) / 2, + }; + // MediaPipe y increases downward, so browMid.y < eyeCenter.y when brows are above eyes. + // Raising brows moves them toward y=0, making this value more negative. + const BRI = (browMid.y - eyeCenter.y) / (interOcular + 1e-6); + + const LCP = + (landmarks[CORNER_LEFT].x + landmarks[CORNER_RIGHT].x) / 2 - neutralLCP; + + return { MAR, EAR, BRI, LCP }; +} + +// ── Gesture classification (ported from backend/sensing/gesture.py) ───────── + +interface Point3D { + x: number; + y: number; + z: number; +} + +function norm3(a: Point3D): number { + return Math.sqrt(a.x ** 2 + a.y ** 2 + a.z ** 2); +} + +function sub3(a: Point3D, b: Point3D): Point3D { + return { x: a.x - b.x, y: a.y - b.y, z: a.z - b.z }; +} + +function scale3(a: Point3D, s: number): Point3D { + return { x: a.x * s, y: a.y * s, z: a.z * s }; +} + +export function classifyGesture(landmarks: Point3D[]): GestureName | null { + const wrist = landmarks[0]; + const palmWidth = + norm3(sub3(landmarks[5], landmarks[17])) + 1e-6; + + const p = landmarks.map((lm) => scale3(sub3(lm, wrist), 1 / palmWidth)); + + const thumbTip = p[4]; + const indexTip = p[8]; + const middleTip = p[12]; + const ringTip = p[16]; + const pinkyTip = p[20]; + const indexMcp = p[5]; + + const fingersCurled = [ + [indexTip, p[5]], + [middleTip, p[9]], + [ringTip, p[13]], + ].every(([tip, mcp]) => norm3(tip) < norm3(mcp)); + + if (thumbTip.y < -0.3 && fingersCurled) return "THUMBS_UP"; + if (thumbTip.y > 0.3 && fingersCurled) return "THUMBS_DOWN"; + + const indexExtended = norm3(indexTip) > norm3(indexMcp) * 1.3; + const othersCurled = [middleTip, ringTip, pinkyTip].every( + (tip) => norm3(tip) < 0.5 + ); + if (indexExtended && othersCurled) return "POINTING"; + + const allExtended = [indexTip, middleTip, ringTip, pinkyTip].every( + (tip) => norm3(tip) > 0.5 + ); + if (allExtended) return "WAVING"; + + return null; +} + +// ── Gaze region mapping (ported from backend/sensing/gaze.py) ──────────────── + +const LEFT_IRIS_CENTER = 468; +const RIGHT_IRIS_CENTER = 473; + +interface GazeRegion { + bounds: [number, number, number, number]; // x_min, y_min, x_max, y_max + bucket: MemoryBucket; +} + +const GAZE_REGIONS: GazeRegion[] = [ + // Centre checked first (most specific region) + { bounds: [0.3, 0.3, 0.7, 0.7], bucket: "social" }, + { bounds: [0.0, 0.0, 0.5, 0.5], bucket: "family" }, + { bounds: [0.5, 0.0, 1.0, 0.5], bucket: "medical" }, + { bounds: [0.0, 0.5, 0.5, 1.0], bucket: "hobbies" }, + { bounds: [0.5, 0.5, 1.0, 1.0], bucket: "daily_routine" }, +]; + +function regionFor(x: number, y: number): MemoryBucket | null { + for (const { bounds, bucket } of GAZE_REGIONS) { + if (x >= bounds[0] && x <= bounds[2] && y >= bounds[1] && y <= bounds[3]) { + return bucket; + } + } + return null; +} + +export class GazeTracker { + private currentRegion: MemoryBucket | null = null; + private dwellStart = 0; + private dwellThresholdMs: number; + + constructor(dwellThresholdMs = 1500) { + this.dwellThresholdMs = dwellThresholdMs; + } + + process(landmarks: { x: number; y: number }[]): MemoryBucket | null { + if (landmarks.length <= RIGHT_IRIS_CENTER) return null; + + const gazeX = + (landmarks[LEFT_IRIS_CENTER].x + landmarks[RIGHT_IRIS_CENTER].x) / 2; + const gazeY = + (landmarks[LEFT_IRIS_CENTER].y + landmarks[RIGHT_IRIS_CENTER].y) / 2; + + const bucket = regionFor(gazeX, gazeY); + + if (bucket !== this.currentRegion) { + this.currentRegion = bucket; + this.dwellStart = performance.now(); + return null; + } + + if ( + bucket !== null && + performance.now() - this.dwellStart >= this.dwellThresholdMs + ) { + this.currentRegion = null; + this.dwellStart = 0; + return bucket; + } + + return null; + } + + reset() { + this.currentRegion = null; + this.dwellStart = 0; + } +} + +// ── Air-writing DTW (ported from backend/sensing/air_writing.py) ───────────── + +const INDEX_TIP = 8; +const VELOCITY_START = 15; +const VELOCITY_END = 5; +const STROKE_GAP_MS = 200; +const RESAMPLE_N = 32; + +function normaliseTrajectory(pts: [number, number][]): [number, number][] { + if (pts.length < 2) return pts; + let minX = Infinity, minY = Infinity, maxX = -Infinity, maxY = -Infinity; + for (const [x, y] of pts) { + minX = Math.min(minX, x); + minY = Math.min(minY, y); + maxX = Math.max(maxX, x); + maxY = Math.max(maxY, y); + } + const scaleX = maxX - minX + 1e-6; + const scaleY = maxY - minY + 1e-6; + const norm = pts.map(([x, y]) => [(x - minX) / scaleX, (y - minY) / scaleY] as [number, number]); + + // Resample to RESAMPLE_N points via linear interpolation + const resampled: [number, number][] = []; + for (let i = 0; i < RESAMPLE_N; i++) { + const t = (i / (RESAMPLE_N - 1)) * (norm.length - 1); + const lo = Math.floor(t); + const hi = Math.min(lo + 1, norm.length - 1); + const frac = t - lo; + resampled.push([ + norm[lo][0] + frac * (norm[hi][0] - norm[lo][0]), + norm[lo][1] + frac * (norm[hi][1] - norm[lo][1]), + ]); + } + return resampled; +} + +function dtwDistance(a: [number, number][], b: [number, number][]): number { + const n = a.length, m = b.length; + const dtw: number[][] = Array.from({ length: n + 1 }, () => + Array(m + 1).fill(Infinity) + ); + dtw[0][0] = 0; + for (let i = 1; i <= n; i++) { + for (let j = 1; j <= m; j++) { + const cost = Math.sqrt( + (a[i - 1][0] - b[j - 1][0]) ** 2 + (a[i - 1][1] - b[j - 1][1]) ** 2 + ); + dtw[i][j] = cost + Math.min(dtw[i - 1][j], dtw[i][j - 1], dtw[i - 1][j - 1]); + } + } + return dtw[n][m]; +} + +export class AirWriter { + private trajectory: [number, number][] = []; + private inStroke = false; + private strokeEndTime = 0; + private prevPt: [number, number] | null = null; + private buffer: string[] = []; + private templates: Map; + + constructor(templates: Map = new Map()) { + this.templates = templates; + } + + processHandLandmarks( + landmarks: { x: number; y: number }[], + frameWidth: number, + frameHeight: number + ): string | null { + const tip: [number, number] = [ + landmarks[INDEX_TIP].x * frameWidth, + landmarks[INDEX_TIP].y * frameHeight, + ]; + + let velocity = 0; + if (this.prevPt) { + velocity = Math.sqrt( + (tip[0] - this.prevPt[0]) ** 2 + (tip[1] - this.prevPt[1]) ** 2 + ); + } + this.prevPt = tip; + + if (velocity > VELOCITY_START) { + this.inStroke = true; + this.trajectory.push(tip); + this.strokeEndTime = 0; + return null; + } + + if (this.inStroke && velocity < VELOCITY_END) { + if (this.strokeEndTime === 0) { + this.strokeEndTime = performance.now(); + } + return this.checkStrokeEnd(); + } + + return null; + } + + private checkStrokeEnd(): string | null { + if (!this.inStroke || this.strokeEndTime === 0) return null; + if (performance.now() - this.strokeEndTime >= STROKE_GAP_MS) { + const char = this.recognise(this.trajectory); + this.trajectory = []; + this.inStroke = false; + this.strokeEndTime = 0; + if (char) this.buffer.push(char); + return char; + } + return null; + } + + private recognise(trajectory: [number, number][]): string | null { + if (trajectory.length < 5 || this.templates.size === 0) return null; + const query = normaliseTrajectory(trajectory); + let bestChar: string | null = null; + let bestDist = Infinity; + for (const [char, template] of this.templates) { + const d = dtwDistance(query, template); + if (d < bestDist) { + bestDist = d; + bestChar = char; + } + } + return bestChar; + } + + getText(): string { + const text = this.buffer.join(""); + this.buffer = []; + return text; + } + + noHand(): string | null { + this.prevPt = null; + return this.checkStrokeEnd(); + } +} diff --git a/frontend/src/main.tsx b/frontend/src/main.tsx new file mode 100644 index 0000000000000000000000000000000000000000..bef5202a32cbd0632c43de40f6e908532903fd42 --- /dev/null +++ b/frontend/src/main.tsx @@ -0,0 +1,10 @@ +import { StrictMode } from 'react' +import { createRoot } from 'react-dom/client' +import './index.css' +import App from './App.tsx' + +createRoot(document.getElementById('root')!).render( + + + , +) diff --git a/frontend/src/types.ts b/frontend/src/types.ts new file mode 100644 index 0000000000000000000000000000000000000000..03bf49a0df593e080887bec354e79cb3cb44dc51 --- /dev/null +++ b/frontend/src/types.ts @@ -0,0 +1,52 @@ +export type Affect = "HAPPY" | "FRUSTRATED" | "NEUTRAL" | "SURPRISED"; +export type GestureName = "THUMBS_UP" | "THUMBS_DOWN" | "POINTING" | "WAVING"; +export type MemoryBucket = "family" | "medical" | "hobbies" | "daily_routine" | "social"; + +export interface SensingState { + affect: Affect | null; + gestureTag: GestureName | null; + gazeBucket: MemoryBucket | null; + airWrittenText: string; +} + +export interface Persona { + id: string; + name: string; + condition: string; + style: string; +} + +export interface ChatRequest { + user_id: string; + query: string; + affect_override: Affect | null; + gesture_tag: GestureName | null; + gaze_bucket: MemoryBucket | null; + air_written_text: string | null; +} + +export interface LatencyLog { + t_sensing: number; + t_intent: number; + t_retrieval: number; + t_generation: number; + t_total: number; +} + +export interface ChatResponse { + user_id: string; + query: string; + response: string; + affect: string; + llm_tier: string; + retrieval_mode: string; + latency: LatencyLog; + guardrail_passed: boolean; +} + +export interface ChatMessage { + role: "partner" | "aac_user"; + content: string; + latency?: LatencyLog; + affect?: string; +} diff --git a/frontend/tsconfig.app.json b/frontend/tsconfig.app.json new file mode 100644 index 0000000000000000000000000000000000000000..1d29c885f28d69fb17925e962f0f309dd7a5352c --- /dev/null +++ b/frontend/tsconfig.app.json @@ -0,0 +1,25 @@ +{ + "compilerOptions": { + "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.app.tsbuildinfo", + "target": "es2023", + "lib": ["ES2023", "DOM", "DOM.Iterable"], + "module": "esnext", + "types": ["vite/client"], + "skipLibCheck": true, + + /* Bundler mode */ + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "verbatimModuleSyntax": true, + "moduleDetection": "force", + "noEmit": true, + "jsx": "react-jsx", + + /* Linting */ + "noUnusedLocals": true, + "noUnusedParameters": true, + "erasableSyntaxOnly": true, + "noFallthroughCasesInSwitch": true + }, + "include": ["src"] +} diff --git a/frontend/tsconfig.json b/frontend/tsconfig.json new file mode 100644 index 0000000000000000000000000000000000000000..1ffef600d959ec9e396d5a260bd3f5b927b2cef8 --- /dev/null +++ b/frontend/tsconfig.json @@ -0,0 +1,7 @@ +{ + "files": [], + "references": [ + { "path": "./tsconfig.app.json" }, + { "path": "./tsconfig.node.json" } + ] +} diff --git a/frontend/tsconfig.node.json b/frontend/tsconfig.node.json new file mode 100644 index 0000000000000000000000000000000000000000..d3c52ea64c6cd6bad118474410f5322f48e257a6 --- /dev/null +++ b/frontend/tsconfig.node.json @@ -0,0 +1,24 @@ +{ + "compilerOptions": { + "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.node.tsbuildinfo", + "target": "es2023", + "lib": ["ES2023"], + "module": "esnext", + "types": ["node"], + "skipLibCheck": true, + + /* Bundler mode */ + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "verbatimModuleSyntax": true, + "moduleDetection": "force", + "noEmit": true, + + /* Linting */ + "noUnusedLocals": true, + "noUnusedParameters": true, + "erasableSyntaxOnly": true, + "noFallthroughCasesInSwitch": true + }, + "include": ["vite.config.ts"] +} diff --git a/frontend/vite.config.ts b/frontend/vite.config.ts new file mode 100644 index 0000000000000000000000000000000000000000..3845fa141bf59338a789d74f5e3465d0d96dd46b --- /dev/null +++ b/frontend/vite.config.ts @@ -0,0 +1,16 @@ +import { defineConfig } from 'vite' +import react from '@vitejs/plugin-react' + +// https://vite.dev/config/ +export default defineConfig({ + plugins: [react()], + server: { + port: 7550, + proxy: { + "/chat": "http://localhost:8000", + "/users": "http://localhost:8000", + "/session": "http://localhost:8000", + "/health": "http://localhost:8000", + }, + }, +}) diff --git a/retrieval/bucket_priors.py b/retrieval/bucket_priors.py deleted file mode 100644 index f2567f60430309b78a207c85195e371277608261..0000000000000000000000000000000000000000 --- a/retrieval/bucket_priors.py +++ /dev/null @@ -1,52 +0,0 @@ -""" -Session-level Bayesian bucket priors (proposal §5.4 Bonus). - -Prior P(bucket_i) is initialized uniformly across the 5 buckets. -After each accepted response, the prior is updated proportionally -to the historical acceptance rate for that bucket in the session. - -P(bucket_i | accept) ∝ P(accept | bucket_i) · P(bucket_i) - -The updated priors are stored in PipelineState and passed to the -retrieval node to bias FAISS search toward the most contextually -likely topic for the session. -""" -from __future__ import annotations - -BUCKETS = ["family", "medical", "hobbies", "daily_routine", "social"] - - -def uniform_priors() -> dict[str, float]: - """Return equal probability mass over all buckets.""" - p = 1.0 / len(BUCKETS) - return {b: p for b in BUCKETS} - - -def update_priors( - priors: dict[str, float], - accepted_bucket: str, - smoothing: float = 0.1, -) -> dict[str, float]: - """ - Bayesian update: boost the accepted bucket, normalise. - - Args: - priors: Current session priors (must sum to ~1.0). - accepted_bucket: Bucket that sourced the accepted response. - smoothing: Additive smoothing constant to prevent zero probabilities. - """ - if not priors: - priors = uniform_priors() - - updated = {b: v + smoothing for b, v in priors.items()} - updated[accepted_bucket] = updated.get(accepted_bucket, smoothing) + 1.0 - - total = sum(updated.values()) - return {b: round(v / total, 6) for b, v in updated.items()} - - -def top_bucket(priors: dict[str, float]) -> str: - """Return the bucket with the highest prior.""" - if not priors: - return BUCKETS[0] - return max(priors, key=priors.get) diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000000000000000000000000000000000000..5589e7d6dda24edfa85d02c5bd29aa2af87aedaf --- /dev/null +++ b/ruff.toml @@ -0,0 +1,9 @@ +[lint] +select = ["E", "F", "W", "I", "UP", "B", "SIM"] +ignore = ["E501", "E741", "B905", "SIM105", "SIM117"] + +[lint.isort] +known-first-party = ["backend"] + +[format] +quote-style = "double" diff --git a/run.sh b/run.sh new file mode 100755 index 0000000000000000000000000000000000000000..b3d54f2ef556186e0556422351599908ca9e2724 --- /dev/null +++ b/run.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +set -euo pipefail + +export PYTHONWARNINGS="ignore::UserWarning:multiprocessing.resource_tracker" + +PIDS=() + +cleanup() { + echo "" + echo "Shutting down..." + for pid in "${PIDS[@]}"; do + kill "$pid" 2>/dev/null + done + # Wait for them to exit, suppress all output + for pid in "${PIDS[@]}"; do + wait "$pid" 2>/dev/null + done + exit 0 +} + +trap cleanup INT TERM + +# Use Node 22 if available (Vite 8 requires Node 20.19+ or 22.12+) +if [ -x /opt/homebrew/opt/node@22/bin/node ]; then + export PATH="/opt/homebrew/opt/node@22/bin:$PATH" +fi + +# Start Ollama if not already running +if command -v ollama >/dev/null 2>&1 && ! curl -s http://localhost:11434/api/tags >/dev/null 2>&1; then + echo "Starting Ollama..." + ollama serve >/dev/null 2>&1 & + PIDS+=($!) + sleep 2 +fi + +echo "Starting FastAPI backend on :8000..." +uvicorn backend.api.main:app --reload --port 8000 2>&1 & +PIDS+=($!) + +# Wait for backend to be reachable before starting frontend +echo "Waiting for backend..." +until curl -s http://localhost:8000/health >/dev/null 2>&1; do + sleep 1 +done +echo "Backend ready." + +echo "Starting React frontend on :7550..." +pnpm --dir frontend dev 2>&1 & +PIDS+=($!) + +echo "All services running. Ctrl+C to stop." +wait diff --git a/setup.sh b/setup.sh index dea685506b7d1f94dbbc2e2840d53b25f2553c6e..794a4de8cb8e66d1efdb2a45c4f09bdf710f5978 100755 --- a/setup.sh +++ b/setup.sh @@ -43,7 +43,7 @@ fi # ── FAISS index build ──────────────────────────────────────────────────────── info "Building FAISS indexes (downloads BGE embedder + reranker on first run)..." -python -m retrieval.vector_store +python -m backend.retrieval.vector_store ok "FAISS indexes built in data/faiss_store/" # ── Ollama model pull ──────────────────────────────────────────────────────── @@ -57,6 +57,15 @@ else ok "Ollama model $LOCAL_MODEL ready" fi +# ── Frontend dependencies ──────────────────────────────────────────────────── +if command -v pnpm >/dev/null 2>&1; then + info "Installing frontend dependencies..." + pnpm --dir frontend install --silent + ok "Frontend dependencies installed" +else + warn "pnpm not found — install it (npm i -g pnpm) then run: pnpm --dir frontend install" +fi + # ── Done ────────────────────────────────────────────────────────────────────── echo "" ok "Setup complete!" @@ -65,9 +74,9 @@ echo " Activate the environment:" echo " conda activate $CONDA_ENV" echo "" echo " Run the CLI:" -echo " python main.py --debug" +echo " python -m backend.main --debug" echo "" echo " Or start the full stack:" -echo " uvicorn api.main:app --reload # FastAPI on :8000" -echo " streamlit run ui/app.py # Streamlit on :8501" +echo " uvicorn backend.api.main:app --reload # FastAPI on :8000" +echo " pnpm --dir frontend dev # React on :7550" echo ""