air-writing

.env.example

@@ -23,3 +23,12 @@ THINKING_MODE=off
 THINKING_TOKEN_BUDGET=4096
 
 FALLBACK_LATENCY_THRESHOLD=3.5
+
+# Vision model used by /ink/recognize (needs image_url support).
+INK_VISION_MODEL=gemini-2.0-flash
+INK_VISION_BASE_URL=https://generativelanguage.googleapis.com/v1beta/openai/
+INK_VISION_API_KEY=
+
+# Frontend flags (VITE_ prefix required for Vite to expose them to the browser).
+# Set to "false" to disable air-writing stroke capture and Gemini ink recognition.
+VITE_AIRWRITING_ENABLED=false

backend/api/main.py

@@ -5,6 +5,7 @@ import re
 import threading
 import time
 from collections import OrderedDict
+from functools import lru_cache
 from pathlib import Path
 
 from fastapi import BackgroundTasks, FastAPI, HTTPException
@@ -979,6 +980,70 @@ def submit_rating(req: RatingRequest):
     return {"status": "ok"}
 
 
+class InkRecognizeRequest(BaseModel):
+    image_base64: str
+
+
+@lru_cache(maxsize=1)
+def _get_vision_client():
+    from openai import OpenAI as _OpenAI
+    return _OpenAI(
+        base_url=settings.ink_vision_base_url,
+        api_key=settings.ink_vision_api_key or "unused",
+    )
+
+
+@app.post("/ink/recognize")
+def ink_recognize(req: InkRecognizeRequest):
+    if not req.image_base64:
+        return {"text": ""}
+    if not settings.ink_vision_api_key:
+        _log.warning("/ink/recognize called but INK_VISION_API_KEY is not set")
+        raise HTTPException(status_code=503, detail="INK_VISION_API_KEY not configured")
+    try:
+        client = _get_vision_client()
+        response = client.chat.completions.create(
+            model=settings.ink_vision_model,
+            messages=[
+                {
+                    "role": "user",
+                    "content": [
+                        {
+                            "type": "image_url",
+                            "image_url": {
+                                "url": f"data:image/png;base64,{req.image_base64}"
+                            },
+                        },
+                        {
+                            "type": "text",
+                            # /no_think suppresses Qwen3 chain-of-thought so the
+                            # answer isn't buried inside <think> tags.
+                            "text": (
+                                "/no_think\n"
+                                "This is a single handwritten character or short word "
+                                "drawn in the air. Reply with ONLY the character or "
+                                "word, nothing else."
+                            ),
+                        },
+                    ],
+                }
+            ],
+            # 512 gives thinking models room to emit <think>…</think> + the answer
+            # before being cut off; the answer itself is stripped out below.
+            max_tokens=512,
+            temperature=0.0,
+        )
+        raw = (response.choices[0].message.content or "")
+        _log.info("/ink/recognize raw → %r", raw[:200])
+        # Strip <think>…</think> blocks emitted by reasoning models (Qwen3 etc.)
+        text = re.sub(r"<think>.*?</think>", "", raw, flags=re.DOTALL).strip()
+        _log.info("/ink/recognize → %r", text)
+        return {"text": text}
+    except Exception as exc:
+        _log.exception("/ink/recognize failed: %r", exc)
+        raise HTTPException(status_code=500, detail=str(exc)) from exc
+
+
 # Serve React frontend — must be last so API routes take priority
 _frontend_dist = Path(__file__).parent.parent.parent / "frontend" / "dist"
 if _frontend_dist.exists():

backend/config/settings.py

@@ -46,6 +46,12 @@ class Settings(BaseSettings):
     # Active tier: "primary" | "fallback"
     active_llm_tier: str = "primary"
 
+    # Vision model used only by /ink/recognize (needs image_url support).
+    # Defaults to Gemini flash via the OpenAI-compatible endpoint.
+    ink_vision_model: str = "gemini-2.0-flash"
+    ink_vision_base_url: str = "https://generativelanguage.googleapis.com/v1beta/openai/"
+    ink_vision_api_key: str = ""
+
     # off | strip | full | suppress
     thinking_mode: str = "off"
     thinking_token_budget: int = 4096

frontend/src/components/SensingStatus.tsx

@@ -58,12 +58,16 @@ export function SensingStatus({ sensing, webcamActive }: Props) {
           </span>
         </div>
       )}
-      {sensing.airWrittenText && (
-        <div className="sensing-row">
-          <span className="sensing-label">Air-writing</span>
-          <span className="sensing-value">{sensing.airWrittenText}</span>
-        </div>
-      )}
+      <div className="sensing-row">
+        <span className="sensing-label">Air-writing</span>
+        <span className="sensing-value">
+          {sensing.airWritingActive
+            ? "✏️ drawing…"
+            : sensing.airWrittenText
+            ? sensing.airWrittenText
+            : "none"}
+        </span>
+      </div>
     </div>
   );
 }

frontend/src/hooks/useSensing.ts

@@ -12,16 +12,20 @@ import {
   AirWriter,
   HeadPoseTracker,
 } from "../lib/sensing";
-import { DEFAULT_AIR_TEMPLATES } from "../lib/airTemplates";
+import { recognizeInkStroke } from "../lib/inkRecognizer";
 
 const GESTURE_DEBOUNCE_FRAMES = 3;
 const AFFECT_DEBOUNCE_FRAMES = 8;
 
+// Set VITE_AIRWRITING_ENABLED=false in .env to disable air-writing.
+const AIRWRITING_ENABLED = import.meta.env.VITE_AIRWRITING_ENABLED !== "false";
+
 export function useSensing() {
   const faceLandmarkerRef = useRef<FaceLandmarker | null>(null);
   const gestureRecognizerRef = useRef<GestureRecognizer | null>(null);
   const gazeTrackerRef = useRef(new GazeTracker());
-  const airWriterRef = useRef(new AirWriter(DEFAULT_AIR_TEMPLATES));
+  const airWriterRef = useRef(new AirWriter());
+  const inkBusyRef = useRef(false);
   const headTrackerRef = useRef(new HeadPoseTracker());
   const calibratePendingRef = useRef(false);
   const headDebugRef = useRef({ dx: 0, dy: 0, maxAbsDx: 0, maxAbsDy: 0, crossings: 0 });
@@ -35,6 +39,7 @@ export function useSensing() {
     gestureTag: null,
     gazeBucket: null,
     airWrittenText: "",
+    airWritingActive: false,
     headSignal: null,
     headCalibrated: false,
     headDebug: { dx: 0, dy: 0, maxAbsDx: 0, maxAbsDy: 0, crossings: 0 },
@@ -133,18 +138,36 @@ export function useSensing() {
       if (gestureResult.gestures && gestureResult.gestures.length > 0) {
         const topGesture = gestureResult.gestures[0][0];
         gestureTag = mapGestureLabel(topGesture.categoryName);
-        const handLandmarks = gestureResult.landmarks[0];
-        airWriterRef.current.processHandLandmarks(
-          handLandmarks,
-          video.videoWidth,
-          video.videoHeight
-        );
-      } else {
+        if (AIRWRITING_ENABLED) {
+          const handLandmarks = gestureResult.landmarks[0];
+          airWriterRef.current.processHandLandmarks(
+            handLandmarks,
+            video.videoWidth,
+            video.videoHeight
+          );
+        }
+      } else if (AIRWRITING_ENABLED) {
         airWriterRef.current.noHand();
       }
 
       const newAirText = airWriterRef.current.getText();
 
+      if (AIRWRITING_ENABLED) {
+        const completedStroke = airWriterRef.current.getCompletedStroke();
+        if (completedStroke && !inkBusyRef.current) {
+          inkBusyRef.current = true;
+          recognizeInkStroke(completedStroke).then((text) => {
+            inkBusyRef.current = false;
+            if (text) {
+              setSensing((prev) => ({
+                ...prev,
+                airWrittenText: prev.airWrittenText + text,
+              }));
+            }
+          });
+        }
+      }
+
       if (gestureTag === gestureCountRef.current.tag) {
         gestureCountRef.current.count++;
       } else {
@@ -170,6 +193,7 @@ export function useSensing() {
         airWrittenText: newAirText
           ? prev.airWrittenText + newAirText
           : prev.airWrittenText,
+        airWritingActive: airWriterRef.current.strokeActive,
         headSignal: headSignal ?? prev.headSignal,
         headCalibrated: headTrackerRef.current.calibrated,
         headDebug: headDebugRef.current,
@@ -201,6 +225,7 @@ export function useSensing() {
       gestureTag: null,
       gazeBucket: null,
       airWrittenText: "",
+      airWritingActive: false,
       headSignal: null,
       headCalibrated: false,
       headDebug: { dx: 0, dy: 0, maxAbsDx: 0, maxAbsDy: 0, crossings: 0 },

frontend/src/lib/inkRecognizer.ts

@@ -0,0 +1,68 @@
+const CANVAS_SIZE = 200;
+const PAD = 20;
+
+function renderStroke(trajectory: [number, number][]): string {
+  const canvas = document.createElement("canvas");
+  canvas.width = CANVAS_SIZE;
+  canvas.height = CANVAS_SIZE;
+  const ctx = canvas.getContext("2d")!;
+
+  ctx.fillStyle = "#ffffff";
+  ctx.fillRect(0, 0, CANVAS_SIZE, CANVAS_SIZE);
+
+  let minX = Infinity, minY = Infinity, maxX = -Infinity, maxY = -Infinity;
+  for (const [x, y] of trajectory) {
+    if (x < minX) minX = x;
+    if (y < minY) minY = y;
+    if (x > maxX) maxX = x;
+    if (y > maxY) maxY = y;
+  }
+
+  const drawArea = CANVAS_SIZE - 2 * PAD;
+  const rangeX = Math.max(maxX - minX, 1);
+  const rangeY = Math.max(maxY - minY, 1);
+  const scale = Math.min(drawArea / rangeX, drawArea / rangeY);
+  const offX = PAD + (drawArea - rangeX * scale) / 2;
+  const offY = PAD + (drawArea - rangeY * scale) / 2;
+
+  ctx.strokeStyle = "#000000";
+  ctx.lineWidth = 5;
+  ctx.lineCap = "round";
+  ctx.lineJoin = "round";
+  ctx.beginPath();
+  const [x0, y0] = trajectory[0];
+  ctx.moveTo((x0 - minX) * scale + offX, (y0 - minY) * scale + offY);
+  for (let i = 1; i < trajectory.length; i++) {
+    const [x, y] = trajectory[i];
+    ctx.lineTo((x - minX) * scale + offX, (y - minY) * scale + offY);
+  }
+  ctx.stroke();
+
+  return canvas.toDataURL("image/png").split(",")[1];
+}
+
+export async function recognizeInkStroke(
+  trajectory: [number, number][]
+): Promise<string> {
+  if (trajectory.length < 5) return "";
+  const base64 = renderStroke(trajectory);
+  try {
+    const res = await fetch("/ink/recognize", {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ image_base64: base64 }),
+    });
+    if (!res.ok) {
+      const body = await res.text().catch(() => "");
+      console.error("[inkRecognizer] HTTP", res.status, body);
+      return "";
+    }
+    const data = await res.json();
+    const text = (data.text as string) || "";
+    console.debug("[inkRecognizer] recognized:", JSON.stringify(text));
+    return text;
+  } catch (err) {
+    console.error("[inkRecognizer] fetch failed:", err);
+    return "";
+  }
+}

frontend/src/lib/sensing.ts

@@ -326,76 +326,25 @@ export class HeadPoseTracker {
   }
 }
 
-// ── Air-writing DTW (ported from backend/sensing/air_writing.py) ─────────────
+// ── Air-writing stroke collector (recognition via Gemini Vision) ─────────────
 
 const INDEX_TIP = 8;
 const VELOCITY_START = 15;
 const VELOCITY_END = 5;
 const STROKE_GAP_MS = 200;
-const RESAMPLE_N = 32;
-
-function normaliseTrajectory(pts: [number, number][]): [number, number][] {
-  if (pts.length < 2) return pts;
-  let minX = Infinity, minY = Infinity, maxX = -Infinity, maxY = -Infinity;
-  for (const [x, y] of pts) {
-    minX = Math.min(minX, x);
-    minY = Math.min(minY, y);
-    maxX = Math.max(maxX, x);
-    maxY = Math.max(maxY, y);
-  }
-  const scaleX = maxX - minX + 1e-6;
-  const scaleY = maxY - minY + 1e-6;
-  const norm = pts.map(([x, y]) => [(x - minX) / scaleX, (y - minY) / scaleY] as [number, number]);
-
-  // Resample to RESAMPLE_N points via linear interpolation
-  const resampled: [number, number][] = [];
-  for (let i = 0; i < RESAMPLE_N; i++) {
-    const t = (i / (RESAMPLE_N - 1)) * (norm.length - 1);
-    const lo = Math.floor(t);
-    const hi = Math.min(lo + 1, norm.length - 1);
-    const frac = t - lo;
-    resampled.push([
-      norm[lo][0] + frac * (norm[hi][0] - norm[lo][0]),
-      norm[lo][1] + frac * (norm[hi][1] - norm[lo][1]),
-    ]);
-  }
-  return resampled;
-}
-
-function dtwDistance(a: [number, number][], b: [number, number][]): number {
-  const n = a.length, m = b.length;
-  const dtw: number[][] = Array.from({ length: n + 1 }, () =>
-    Array(m + 1).fill(Infinity)
-  );
-  dtw[0][0] = 0;
-  for (let i = 1; i <= n; i++) {
-    for (let j = 1; j <= m; j++) {
-      const cost = Math.sqrt(
-        (a[i - 1][0] - b[j - 1][0]) ** 2 + (a[i - 1][1] - b[j - 1][1]) ** 2
-      );
-      dtw[i][j] = cost + Math.min(dtw[i - 1][j], dtw[i][j - 1], dtw[i - 1][j - 1]);
-    }
-  }
-  return dtw[n][m];
-}
 
 export class AirWriter {
   private trajectory: [number, number][] = [];
   private inStroke = false;
   private strokeEndTime = 0;
   private prevPt: [number, number] | null = null;
-  private buffer: string[] = [];
-  private templates: Map<string, [number, number][]>;
-
-  constructor(templates: Map<string, [number, number][]> = new Map()) {
-    this.templates = templates;
-  }
+  private pendingStroke: [number, number][] | null = null;
 
   processHandLandmarks(
     landmarks: { x: number; y: number }[],
     frameWidth: number,
     frameHeight: number
-  ): string | null {
+  ): void {
     const tip: [number, number] = [
       landmarks[INDEX_TIP].x * frameWidth,
       landmarks[INDEX_TIP].y * frameHeight,
@@ -413,70 +362,50 @@ export class AirWriter {
       this.inStroke = true;
       this.trajectory.push(tip);
       this.strokeEndTime = 0;
-      return null;
+      return;
     }
 
     if (this.inStroke && velocity < VELOCITY_END) {
       if (this.strokeEndTime === 0) {
         this.strokeEndTime = performance.now();
       }
-      return this.checkStrokeEnd();
+      this.checkStrokeEnd();
     }
-
-    return null;
   }
 
-  private checkStrokeEnd(): string | null {
-    if (!this.inStroke || this.strokeEndTime === 0) return null;
+  private checkStrokeEnd(): void {
+    if (!this.inStroke || this.strokeEndTime === 0) return;
     if (performance.now() - this.strokeEndTime >= STROKE_GAP_MS) {
-      const char = this.recognise(this.trajectory);
+      if (this.trajectory.length >= 5) {
+        this.pendingStroke = [...this.trajectory];
+      }
       this.trajectory = [];
       this.inStroke = false;
       this.strokeEndTime = 0;
-      if (char) this.buffer.push(char);
-      return char;
     }
-    return null;
   }
 
-  private recognise(trajectory: [number, number][]): string | null {
-    if (trajectory.length < 5) {
-      return null;
-    }
-    if (this.templates.size === 0) {
-      console.debug("[AirWriter] stroke completed but template bank is empty");
-      return null;
-    }
-    const query = normaliseTrajectory(trajectory);
-    let bestChar: string | null = null;
-    let bestDist = Infinity;
-    for (const [char, template] of this.templates) {
-      const d = dtwDistance(query, template);
-      if (d < bestDist) {
-        bestDist = d;
-        bestChar = char;
-      }
-    }
-    // Reject poor matches so we don't pass garbage to the LLM.
-    // Threshold is empirical — tune once real users test this.
-    const MATCH_THRESHOLD = 8.0;
-    if (bestDist > MATCH_THRESHOLD) {
-      console.debug(
-        `[AirWriter] no template matched (best='${bestChar}', dist=${bestDist.toFixed(2)})`
-      );
-      return null;
-    }
-    return bestChar;
+  get strokeActive(): boolean {
+    return this.inStroke;
   }
 
+  // Returns the completed stroke trajectory and clears it (call once per frame).
+  getCompletedStroke(): [number, number][] | null {
+    const s = this.pendingStroke;
+    this.pendingStroke = null;
+    return s;
+  }
+
+  // Kept for API compatibility — always returns "".
   getText(): string {
-    const text = this.buffer.join("");
-    this.buffer = [];
-    return text;
+    return "";
   }
 
-  noHand(): string | null {
+  noHand(): void {
+    if (this.inStroke && this.strokeEndTime === 0) {
+      this.strokeEndTime = performance.now();
+    }
     this.prevPt = null;
-    return this.checkStrokeEnd();
+    this.checkStrokeEnd();
   }
 }

frontend/src/types.ts

@@ -16,6 +16,7 @@ export interface SensingState {
   gestureTag: GestureName | null;
   gazeBucket: MemoryBucket | null;
   airWrittenText: string;
+  airWritingActive: boolean;
   headSignal: HeadSignal | null;
   headCalibrated: boolean;
   headDebug: HeadDebug;

frontend/vite.config.ts

@@ -4,6 +4,7 @@ import react from '@vitejs/plugin-react'
 // https://vite.dev/config/
 export default defineConfig({
   plugins: [react()],
+  envDir: "..",   // read .env from repo root (shared with backend)
   server: {
     port: 7550,
     proxy: {
@@ -14,6 +15,7 @@ export default defineConfig({
       "/evals": "http://localhost:8000",
       "/feedback": "http://localhost:8000",
       "/debug": "http://localhost:8000",
+      "/ink": "http://localhost:8000",
     },
   },
 })