added affect and gesture fix

backend/evals/multimodal_alignment.py

@@ -40,7 +40,9 @@ _AFFECT_TARGET = {
 _GESTURE_OPENER_PATTERNS = {
     "THUMBS_UP": re.compile(r"^\s*(yes|yeah|totally|for sure|absolutely|sure)\b", re.I),
     "THUMBS_DOWN": re.compile(r"^\s*(no|nah|not really|i'd rather not)\b", re.I),
-    "WAVING": re.compile(r"^\s*(hi|hey|hello)\b", re.I),
+    "OPEN_PALM": re.compile(r"^\s*(hi|hey|hello)\b", re.I),
+    "VICTORY": re.compile(r"^\s*(yes|awesome|great|fantastic|amazing|woo)\b", re.I),
+    "I_LOVE_YOU": re.compile(r"^\s*(love|i love|adore|care)\b", re.I),
 }
 
 

backend/pipeline/nodes/planner.py

@@ -659,7 +659,7 @@ def _format_multimodal_intent(
         if source == "conflict_air" and air_t:
             return (
                 f'\nThe user spoke "{voice_t}" but also air-wrote "{air_t}". '
-                "The air-written token is a canonical AAC signal "
+                "The air-writing token is a canonical AAC signal "
                 "(help/stop/water/done/more) — prioritise it over the spoken "
                 "words, which may have been misheard."
             )

backend/pipeline/state.py

@@ -91,7 +91,7 @@ class PipelineState(TypedDict):
     affect: AffectState | None
     gesture_tag: str | None  # e.g. "THUMBS_UP"
     gaze_bucket: str | None  # bucket hinted by gaze fixation
-    air_written_text: str | None  # concatenated air-written chars
+    air_written_text: str | None  # concatenated air-writing chars
     head_signal: str | None  # "HEAD_SHAKE" | "HEAD_NOD_DISSATISFIED"
     voice_text: str | None  # raw Web Speech transcript, pre-resolution
     # Resolved voice⇄air-writing intent. Keys: text, source, voice_text, air_text.

backend/sensing/labels.py

@@ -7,12 +7,24 @@ GESTURE_DIRECTIVES: dict[str, dict[str, str]] = {
         "tone": "[GESTURE:THUMBS_DOWN][TONE:NEGATIVE]",
         "opener_hint": "Open by declining or disagreeing briefly.",
     },
-    "POINTING": {
-        "tone": "[GESTURE:POINTING][INTENT:REFERENTIAL]",
+    "POINTING_UP": {
+        "tone": "[GESTURE:POINTING_UP][INTENT:REFERENTIAL]",
         "opener_hint": "Treat the query as referring to a specific named thing.",
     },
-    "WAVING": {
-        "tone": "[GESTURE:WAVING][INTENT:GREETING]",
-        "opener_hint": "Open with a greeting.",
+    "CLOSED_FIST": {
+        "tone": "[GESTURE:CLOSED_FIST][TONE:EMPHATIC]",
+        "opener_hint": "Respond with emphasis or urgency — something important needs saying.",
+    },
+    "OPEN_PALM": {
+        "tone": "[GESTURE:OPEN_PALM][INTENT:GREETING]",
+        "opener_hint": "Open with a warm greeting.",
+    },
+    "VICTORY": {
+        "tone": "[GESTURE:VICTORY][TONE:CELEBRATORY]",
+        "opener_hint": "Open with celebration or excitement.",
+    },
+    "I_LOVE_YOU": {
+        "tone": "[GESTURE:I_LOVE_YOU][TONE:AFFECTIONATE]",
+        "opener_hint": "Open with warmth and affection.",
     },
 }

frontend/src/components/SensingStatus.tsx

@@ -60,7 +60,7 @@ export function SensingStatus({ sensing, webcamActive }: Props) {
       )}
       {sensing.airWrittenText && (
         <div className="sensing-row">
-          <span className="sensing-label">Air-written</span>
+          <span className="sensing-label">Air-writing</span>
           <span className="sensing-value">{sensing.airWrittenText}</span>
         </div>
       )}

frontend/src/hooks/useSensing.ts

@@ -1,35 +1,30 @@
 import { useRef, useCallback, useState, useEffect } from "react";
 import {
   FaceLandmarker,
-  HandLandmarker,
+  GestureRecognizer,
   FilesetResolver,
 } from "@mediapipe/tasks-vision";
 import type { SensingState } from "../types";
 import {
-  computeAffectVector,
   classifyAffect,
-  classifyGesture,
+  mapGestureLabel,
   GazeTracker,
   AirWriter,
   HeadPoseTracker,
 } from "../lib/sensing";
 import { DEFAULT_AIR_TEMPLATES } from "../lib/airTemplates";
 
-const EMA_ALPHA = 0.2;
 const GESTURE_DEBOUNCE_FRAMES = 3;
 const AFFECT_DEBOUNCE_FRAMES = 8;
 
 export function useSensing() {
   const faceLandmarkerRef = useRef<FaceLandmarker | null>(null);
-  const handLandmarkerRef = useRef<HandLandmarker | null>(null);
+  const gestureRecognizerRef = useRef<GestureRecognizer | null>(null);
   const gazeTrackerRef = useRef(new GazeTracker());
   const airWriterRef = useRef(new AirWriter(DEFAULT_AIR_TEMPLATES));
   const headTrackerRef = useRef(new HeadPoseTracker());
   const calibratePendingRef = useRef(false);
   const headDebugRef = useRef({ dx: 0, dy: 0, maxAbsDx: 0, maxAbsDy: 0, crossings: 0 });
-  const neutralLCPRef = useRef<number | null>(null);
-  const calibBufferRef = useRef<number[]>([]);
-  const smoothedRef = useRef({ MAR: 0, EAR: 0.3, BRI: -0.3, LCP: 0 });
   const gestureCountRef = useRef<{ tag: SensingState["gestureTag"]; count: number }>({ tag: null, count: 0 });
   const affectCountRef = useRef<{ affect: SensingState["affect"]; count: number }>({ affect: null, count: 0 });
   const initingRef = useRef(false);
@@ -49,9 +44,9 @@ export function useSensing() {
   useEffect(() => {
     return () => {
       faceLandmarkerRef.current?.close();
-      handLandmarkerRef.current?.close();
+      gestureRecognizerRef.current?.close();
       faceLandmarkerRef.current = null;
-      handLandmarkerRef.current = null;
+      gestureRecognizerRef.current = null;
     };
   }, []);
 
@@ -72,16 +67,16 @@ export function useSensing() {
           },
           runningMode: "VIDEO",
           numFaces: 1,
-          outputFaceBlendshapes: false,
+          outputFaceBlendshapes: true,
           outputFacialTransformationMatrixes: false,
         }
       );
-      handLandmarkerRef.current = await HandLandmarker.createFromOptions(
+      gestureRecognizerRef.current = await GestureRecognizer.createFromOptions(
         vision,
         {
           baseOptions: {
             modelAssetPath:
-              "https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task",
+              "https://storage.googleapis.com/mediapipe-models/gesture_recognizer/gesture_recognizer/float16/1/gesture_recognizer.task",
             delegate: "GPU",
           },
           runningMode: "VIDEO",
@@ -103,8 +98,8 @@ export function useSensing() {
   const processFrame = useCallback(
     (video: HTMLVideoElement, timestamp: number) => {
       const faceLandmarker = faceLandmarkerRef.current;
-      const handLandmarker = handLandmarkerRef.current;
-      if (!faceLandmarker || !handLandmarker) return;
+      const gestureRecognizer = gestureRecognizerRef.current;
+      if (!faceLandmarker || !gestureRecognizer) return;
 
       let affect: SensingState["affect"] = null;
       let gazeBucket: SensingState["gazeBucket"] = null;
@@ -114,38 +109,17 @@ export function useSensing() {
       if (faceResult.faceLandmarks && faceResult.faceLandmarks.length > 0) {
         const landmarks = faceResult.faceLandmarks[0];
 
-        // Average the raw LCP (vertical corner pull, pre-offset) over ~30 frames
-        // of the user's face before locking neutral. Single-frame calibration is
-        // too noisy and tended to bake in a momentary smile as "neutral".
-        // During calibration, affect stays null but gaze/head/gesture still flow.
-        if (neutralLCPRef.current === null) {
-          const raw0 = computeAffectVector(landmarks, 0);
-          calibBufferRef.current.push(raw0.LCP);
-          if (calibBufferRef.current.length >= 30) {
-            const sum = calibBufferRef.current.reduce((a, b) => a + b, 0);
-            neutralLCPRef.current = sum / calibBufferRef.current.length;
-            calibBufferRef.current = [];
-          }
-        }
-
         if (calibratePendingRef.current) {
           headTrackerRef.current.calibrate(landmarks);
           calibratePendingRef.current = false;
         }
 
-        if (neutralLCPRef.current !== null) {
-          const raw = computeAffectVector(landmarks, neutralLCPRef.current);
-
-          const prev = smoothedRef.current;
-          const smoothed = {
-            MAR: EMA_ALPHA * raw.MAR + (1 - EMA_ALPHA) * prev.MAR,
-            EAR: EMA_ALPHA * raw.EAR + (1 - EMA_ALPHA) * prev.EAR,
-            BRI: EMA_ALPHA * raw.BRI + (1 - EMA_ALPHA) * prev.BRI,
-            LCP: EMA_ALPHA * raw.LCP + (1 - EMA_ALPHA) * prev.LCP,
-          };
-          smoothedRef.current = smoothed;
-
-          affect = classifyAffect(smoothed);
+        if (faceResult.faceBlendshapes && faceResult.faceBlendshapes.length > 0) {
+          const bs: Record<string, number> = {};
+          for (const cat of faceResult.faceBlendshapes[0].categories) {
+            bs[cat.categoryName] = cat.score;
+          }
+          affect = classifyAffect(bs);
         }
 
         gazeBucket = gazeTrackerRef.current.process(landmarks);
@@ -155,10 +129,11 @@ export function useSensing() {
 
       let gestureTag: SensingState["gestureTag"] = null;
 
-      const handResult = handLandmarker.detectForVideo(video, timestamp);
-      if (handResult.landmarks && handResult.landmarks.length > 0) {
-        const handLandmarks = handResult.landmarks[0];
-        gestureTag = classifyGesture(handLandmarks);
+      const gestureResult = gestureRecognizer.recognizeForVideo(video, timestamp);
+      if (gestureResult.gestures && gestureResult.gestures.length > 0) {
+        const topGesture = gestureResult.gestures[0][0];
+        gestureTag = mapGestureLabel(topGesture.categoryName);
+        const handLandmarks = gestureResult.landmarks[0];
         airWriterRef.current.processHandLandmarks(
           handLandmarks,
           video.videoWidth,
@@ -217,9 +192,6 @@ export function useSensing() {
   }, []);
 
   const resetCalibration = useCallback(() => {
-    neutralLCPRef.current = null;
-    calibBufferRef.current = [];
-    smoothedRef.current = { MAR: 0, EAR: 0.3, BRI: -0.3, LCP: 0 };
     gestureCountRef.current = { tag: null, count: 0 };
     affectCountRef.current = { affect: null, count: 0 };
     gazeTrackerRef.current.reset();

frontend/src/lib/resolveIntent.ts

@@ -2,7 +2,7 @@ import { DEFAULT_AIR_TEMPLATES } from "./airTemplates";
 
 // Canonical AAC tokens that carry high signal when someone air-writes them —
 // short, action-oriented, and hard to confuse for casual chat. When the
-// voice transcript and the air-written text disagree, these tokens win.
+// voice transcript and the air-writing text disagree, these tokens win.
 const AAC_PRIORITY_TOKENS: ReadonlySet<string> = new Set(
   ["help", "stop", "water", "done", "more"].filter((t) =>
     DEFAULT_AIR_TEMPLATES.has(t)

frontend/src/lib/sensing.ts

@@ -1,132 +1,37 @@
 import type { Affect, GestureName, MemoryBucket } from "../types";
 
-// ── Affect classification (ported from backend/sensing/face_mesh.py) ────────
-
-interface AffectVector {
-  MAR: number;
-  EAR: number;
-  BRI: number;
-  LCP: number;
-}
-
-export function classifyAffect(v: AffectVector): Affect {
-  // BRI is relative (browMid.y - eyeCenter.y) / interOcular — more negative = brows raised higher
-  // LCP is vertical offset of lip corners from mouth center, normalised by inter-ocular,
-  //   relative to calibrated neutral — positive = corners pulled UP (smile), negative = DOWN (frown)
-  // MAR is absolute ratio — higher = mouth more open
-  // EAR is absolute ratio — lower = eyes more closed / squinting
-  if (v.BRI < -0.35 && v.MAR > 0.4) return "SURPRISED";
-  // FRUSTRATED: a clear frown, OR brows lowered + squinting — either signals displeasure
-  if (v.LCP < -0.018) return "FRUSTRATED";
-  if (v.BRI > -0.2 && v.EAR < 0.18) return "FRUSTRATED";
-  if (v.LCP > 0.012) return "HAPPY";
+// ── Affect classification via MediaPipe blendshapes ──────────────────────────
+
+export function classifyAffect(bs: Record<string, number>): Affect {
+  const smileLeft   = bs["mouthSmileLeft"]  ?? 0;
+  const smileRight  = bs["mouthSmileRight"] ?? 0;
+  const browDownL   = bs["browDownLeft"]    ?? 0;
+  const browDownR   = bs["browDownRight"]   ?? 0;
+  const squintL     = bs["eyeSquintLeft"]   ?? 0;
+  const squintR     = bs["eyeSquintRight"]  ?? 0;
+  const jawOpen     = bs["jawOpen"]         ?? 0;
+  const browInnerUp = bs["browInnerUp"]     ?? 0;
+
+  if (jawOpen > 0.4 && browInnerUp > 0.5) return "SURPRISED";
+  if (browDownL > 0.4 || browDownR > 0.4) return "FRUSTRATED";
+  if (squintL > 0.5 && squintR > 0.5)     return "FRUSTRATED";
+  if (smileLeft > 0.5 && smileRight > 0.5) return "HAPPY";
   return "NEUTRAL";
 }
 
-// Face landmark indices (MediaPipe 478-point mesh)
-const MOUTH_TOP = 13, MOUTH_BOTTOM = 14, MOUTH_LEFT = 61, MOUTH_RIGHT = 291;
-const EYE_TOP = 159, EYE_BOTTOM = 145, EYE_LEFT = 33, EYE_RIGHT = 133;
-const BROW_LEFT = 70, BROW_RIGHT = 300;
-const CORNER_LEFT = 61, CORNER_RIGHT = 291;
-
-function dist(a: { x: number; y: number }, b: { x: number; y: number }): number {
-  return Math.sqrt((a.x - b.x) ** 2 + (a.y - b.y) ** 2);
-}
-
-export function computeAffectVector(
-  landmarks: { x: number; y: number }[],
-  neutralLCP: number
-): AffectVector {
-  const MAR =
-    dist(landmarks[MOUTH_TOP], landmarks[MOUTH_BOTTOM]) /
-    (dist(landmarks[MOUTH_LEFT], landmarks[MOUTH_RIGHT]) + 1e-6);
-
-  const EAR =
-    dist(landmarks[EYE_TOP], landmarks[EYE_BOTTOM]) /
-    (dist(landmarks[EYE_LEFT], landmarks[EYE_RIGHT]) + 1e-6);
-
-  const eyeCenter = {
-    x: (landmarks[EYE_LEFT].x + landmarks[EYE_RIGHT].x) / 2,
-    y: (landmarks[EYE_LEFT].y + landmarks[EYE_RIGHT].y) / 2,
-  };
-  const interOcular = dist(landmarks[EYE_LEFT], landmarks[EYE_RIGHT]);
-  const browMid = {
-    x: (landmarks[BROW_LEFT].x + landmarks[BROW_RIGHT].x) / 2,
-    y: (landmarks[BROW_LEFT].y + landmarks[BROW_RIGHT].y) / 2,
-  };
-  // MediaPipe y increases downward, so browMid.y < eyeCenter.y when brows are above eyes.
-  // Raising brows moves them toward y=0, making this value more negative.
-  const BRI = (browMid.y - eyeCenter.y) / (interOcular + 1e-6);
-
-  // Lip-corner pull: average y of the two corners vs. mouth vertical centre,
-  // normalised by inter-ocular distance, relative to calibrated neutral.
-  // MediaPipe y increases downward, so corners rising above the mouth centre → negative raw,
-  // which we flip so smile = positive. Subtracting the calibrated neutral removes per-face bias.
-  const mouthCentreY = (landmarks[MOUTH_TOP].y + landmarks[MOUTH_BOTTOM].y) / 2;
-  const cornerAvgY = (landmarks[CORNER_LEFT].y + landmarks[CORNER_RIGHT].y) / 2;
-  const rawLCP = (mouthCentreY - cornerAvgY) / (interOcular + 1e-6);
-  const LCP = rawLCP - neutralLCP;
-
-  return { MAR, EAR, BRI, LCP };
-}
-
-// ── Gesture classification (ported from backend/sensing/gesture.py) ─────────
-
-interface Point3D {
-  x: number;
-  y: number;
-  z: number;
-}
-
-function norm3(a: Point3D): number {
-  return Math.sqrt(a.x ** 2 + a.y ** 2 + a.z ** 2);
-}
-
-function sub3(a: Point3D, b: Point3D): Point3D {
-  return { x: a.x - b.x, y: a.y - b.y, z: a.z - b.z };
-}
-
-function scale3(a: Point3D, s: number): Point3D {
-  return { x: a.x * s, y: a.y * s, z: a.z * s };
-}
-
-export function classifyGesture(landmarks: Point3D[]): GestureName | null {
-  const wrist = landmarks[0];
-  const palmWidth =
-    norm3(sub3(landmarks[5], landmarks[17])) + 1e-6;
-
-  const p = landmarks.map((lm) => scale3(sub3(lm, wrist), 1 / palmWidth));
-
-  const thumbTip = p[4];
-  const indexTip = p[8];
-  const middleTip = p[12];
-  const ringTip = p[16];
-  const pinkyTip = p[20];
-  const indexMcp = p[5];
-
-  const fingersCurled = [
-    [indexTip, p[5]],
-    [middleTip, p[9]],
-    [ringTip, p[13]],
-  ].every(([tip, mcp]) => norm3(tip) < norm3(mcp));
-
-  // Check POINTING before THUMBS_UP — pointing with a raised thumb would otherwise
-  // satisfy fingersCurled on a noisy frame and fire the wrong label first.
-  const indexExtended = norm3(indexTip) > norm3(indexMcp) * 1.3;
-  const othersCurled = [middleTip, ringTip, pinkyTip].every(
-    (tip) => norm3(tip) < 0.7
-  );
-  if (indexExtended && othersCurled) return "POINTING";
-
-  if (thumbTip.y < -0.3 && fingersCurled) return "THUMBS_UP";
-  if (thumbTip.y > 0.3 && fingersCurled) return "THUMBS_DOWN";
-
-  const allExtended = [indexTip, middleTip, ringTip, pinkyTip, thumbTip].every(
-    (tip) => norm3(tip) > 0.7
-  );
-  if (allExtended) return "WAVING";
-
-  return null;
+// ── Gesture label mapping from MediaPipe GestureRecognizer ───────────────────
+
+export function mapGestureLabel(label: string): GestureName | null {
+  switch (label) {
+    case "Thumb_Up":    return "THUMBS_UP";
+    case "Thumb_Down":  return "THUMBS_DOWN";
+    case "Pointing_Up": return "POINTING_UP";
+    case "Closed_Fist": return "CLOSED_FIST";
+    case "Open_Palm":   return "OPEN_PALM";
+    case "Victory":     return "VICTORY";
+    case "ILoveYou":    return "I_LOVE_YOU";
+    default:            return null;
+  }
 }
 
 // ── Gaze region mapping (ported from backend/sensing/gaze.py) ────────────────

frontend/src/types.ts

@@ -1,5 +1,5 @@
 export type Affect = "HAPPY" | "FRUSTRATED" | "NEUTRAL" | "SURPRISED";
-export type GestureName = "THUMBS_UP" | "THUMBS_DOWN" | "POINTING" | "WAVING";
+export type GestureName = "THUMBS_UP" | "THUMBS_DOWN" | "POINTING_UP" | "CLOSED_FIST" | "OPEN_PALM" | "VICTORY" | "I_LOVE_YOU";
 export type MemoryBucket = "family" | "medical" | "hobbies" | "daily_routine" | "social";
 export type HeadSignal = "HEAD_SHAKE" | "HEAD_NOD_DISSATISFIED";