Upload frontend/src/MediaPipeProcessor.js with huggingface_hub

frontend/src/MediaPipeProcessor.js

@@ -0,0 +1,582 @@
+/**
+ * MediaPipeProcessor.js
+ * 
+ * Advanced hand gesture recognition with swipe and pinch detection.
+ * Integrates with LLM Flow for gesture-triggered actions.
+ * 
+ * Features:
+ * - Hand landmark detection (21 points)
+ * - Face mesh detection (468 points) for face blur
+ * - Swipe gesture detection (left, right, up, down)
+ * - Pinch gesture detection
+ * - Finger count detection
+ * - Real-time gesture-to-action triggering
+ */
+
+class MediaPipeProcessor {
+  constructor(options = {}) {
+    this.options = {
+      maxHands: options.maxHands || 1,
+      modelComplexity: options.modelComplexity || 1,
+      minDetectionConfidence: options.minDetectionConfidence || 0.5,
+      minTrackingConfidence: options.minTrackingConfidence || 0.5,
+      onGestureDetected: options.onGestureDetected || null,
+      onLandmarksUpdate: options.onLandmarksUpdate || null,
+      onSwipeDetected: options.onSwipeDetected || null,
+      onPinchDetected: options.onPinchDetected || null,
+      ...options
+    };
+
+    this.hands = null;
+    this.faceMesh = null;
+    this.isInitialized = false;
+    this.camera = null;
+
+    this.lastLandmarks = null;
+    this.landmarkHistory = [];
+    this.maxHistorySize = 30;
+
+    this.swipeDetector = new SwipeDetector();
+    this.pinchDetector = new PinchDetector();
+    this.fingerCounter = new FingerCounter();
+
+    this.gestureCallback = null;
+    this.apiUrl = options.apiUrl || 'http://localhost:5001/api';
+  }
+
+  async initialize() {
+    if (this.isInitialized) return true;
+
+    try {
+      const { Hands } = await import('https://cdn.jsdelivr.net/npm/@mediapipe/hands@0.4.1675469240/hands.js');
+      const { FaceMesh } = await import('https://cdn.jsdelivr.net/npm/@mediapipe/face_mesh@0.4.1633529614/face_mesh.js');
+      const { Camera } = await import('https://cdn.jsdelivr.net/npm/@mediapipe/camera_utils@0.3.1640029074/camera_utils.js');
+
+      this.hands = new Hands({
+        locateFile: (file) => `https://cdn.jsdelivr.net/npm/@mediapipe/hands@0.4.1675469240/${file}`
+      });
+
+      this.hands.setOptions({
+        maxNumHands: this.options.maxHands,
+        modelComplexity: this.options.modelComplexity,
+        minDetectionConfidence: this.options.minDetectionConfidence,
+        minTrackingConfidence: this.options.minTrackingConfidence
+      });
+
+      this.hands.onResults((results) => this.onHandResults(results));
+
+      this.faceMesh = new FaceMesh({
+        locateFile: (file) => `https://cdn.jsdelivr.net/npm/@mediapipe/face_mesh@0.4.1633529614/${file}`
+      });
+
+      this.faceMesh.setOptions({
+        maxNumFaces: 1,
+        refineLandmarks: true,
+        minDetectionConfidence: 0.5,
+        minTrackingConfidence: 0.5
+      });
+
+      this.faceMesh.onResults((results) => this.onFaceResults(results));
+
+      this.isInitialized = true;
+      return true;
+    } catch (error) {
+      console.error('Failed to initialize MediaPipe:', error);
+      return false;
+    }
+  }
+
+  async startCamera(videoElement, canvasElement, overlayCanvasElement) {
+    if (!this.isInitialized) {
+      await this.initialize();
+    }
+
+    this.videoElement = videoElement;
+    this.canvasElement = canvasElement;
+    this.overlayCanvasElement = overlayCanvasElement;
+
+    try {
+      const stream = await navigator.mediaDevices.getUserMedia({
+        video: { facingMode: 'user', width: 640, height: 480 }
+      });
+
+      videoElement.srcObject = stream;
+      await videoElement.play();
+
+      this.camera = new Camera(videoElement, {
+        onFrame: async () => {
+          if (this.hands && this.faceMesh) {
+            await this.hands.send({ image: videoElement });
+            await this.faceMesh.send({ image: videoElement });
+          }
+        },
+        width: 640,
+        height: 480
+      });
+
+      await this.camera.start();
+      return true;
+    } catch (error) {
+      console.error('Failed to start camera:', error);
+      return false;
+    }
+  }
+
+  stopCamera() {
+    if (this.camera) {
+      this.camera.stop();
+      this.camera = null;
+    }
+
+    if (this.videoElement && this.videoElement.srcObject) {
+      this.videoElement.srcObject.getTracks().forEach(track => track.stop());
+      this.videoElement.srcObject = null;
+    }
+  }
+
+  onHandResults(results) {
+    if (results.multiHandLandmarks && results.multiHandLandmarks.length > 0) {
+      const landmarks = results.multiHandLandmarks[0];
+      this.lastLandmarks = landmarks;
+      
+      this.landmarkHistory.push({
+        landmarks: landmarks.map(lm => [lm.x, lm.y, lm.z]),
+        timestamp: Date.now()
+      });
+      
+      if (this.landmarkHistory.length > this.maxHistorySize) {
+        this.landmarkHistory.shift();
+      }
+
+      if (this.options.onLandmarksUpdate) {
+        this.options.onLandmarksUpdate(landmarks);
+      }
+
+      const fingerCount = this.fingerCounter.count(landmarks);
+      const swipe = this.swipeDetector.detect(landmarks, fingerCount, this.landmarkHistory);
+      const pinch = this.pinchDetector.detect(landmarks);
+
+      if (swipe) {
+        this.onSwipeGesture(swipe, fingerCount);
+      }
+
+      if (pinch) {
+        this.onPinchGesture(pinch);
+      }
+
+      const basicGesture = this.detectBasicGesture(landmarks, fingerCount);
+      if (basicGesture && this.options.onGestureDetected) {
+        this.options.onGestureDetected(basicGesture);
+      }
+
+      this.drawHandLandmarks(landmarks);
+    } else {
+      this.lastLandmarks = null;
+      this.clearOverlay();
+    }
+  }
+
+  onFaceResults(results) {
+    if (results.multiFaceLandmarks && results.multiFaceLandmarks.length > 0) {
+      this.applyFaceBlur(results.multiFaceLandmarks[0]);
+    }
+  }
+
+  onSwipeGesture(swipe, fingerCount) {
+    const gestureName = `${fingerCount}_finger_swipe_${swipe.direction}`;
+    
+    console.log(`Swipe detected: ${gestureName} (speed: ${swipe.speed.toFixed(3)})`);
+
+    if (this.options.onSwipeDetected) {
+      this.options.onSwipeDetected({
+        ...swipe,
+        fingerCount,
+        gestureName
+      });
+    }
+
+    this.sendGestureToBackend(gestureName, {
+      direction: swipe.direction,
+      fingerCount,
+      speed: swipe.speed,
+      startPosition: swipe.startPosition,
+      endPosition: swipe.endPosition
+    });
+  }
+
+  onPinchGesture(pinch) {
+    const gestureName = `pinch_${pinch.type}`;
+    
+    console.log(`Pinch detected: ${gestureName} (distance: ${pinch.distance.toFixed(3)})`);
+
+    if (this.options.onPinchDetected) {
+      this.options.onPinchDetected(pinch);
+    }
+
+    this.sendGestureToBackend(gestureName, {
+      type: pinch.type,
+      distance: pinch.distance,
+      thumbTip: pinch.thumbTip,
+      indexTip: pinch.indexTip
+    });
+  }
+
+  detectBasicGesture(landmarks, fingerCount) {
+    if (fingerCount >= 4) {
+      return { type: 'open_palm', fingerCount };
+    }
+    
+    if (fingerCount === 1) {
+      const indexTip = landmarks[8];
+      const indexBase = landmarks[5];
+      if (indexTip.y < indexBase.y) {
+        return { type: 'pointing', fingerCount: 1 };
+      }
+    }
+
+    if (this.isThumbsUp(landmarks)) {
+      return { type: 'thumbs_up', fingerCount };
+    }
+
+    return null;
+  }
+
+  isThumbsUp(landmarks) {
+    const thumbTip = landmarks[4];
+    const thumbIP = landmarks[3];
+    const indexTip = landmarks[8];
+    const middleTip = landmarks[12];
+    
+    const thumbUp = thumbTip.y < thumbIP.y;
+    const fingersDown = indexTip.y > landmarks[6].y && 
+                        middleTip.y > landmarks[10].y;
+    
+    return thumbUp && fingersDown;
+  }
+
+  sendGestureToBackend(gestureName, parameters) {
+    if (!this.lastLandmarks) return;
+
+    const landmarksArray = this.lastLandmarks.map(lm => [lm.x, lm.y, lm.z]);
+
+    fetch(`${this.apiUrl}/llm/gesture-action`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({
+        user_id: 'demo',
+        landmarks: landmarksArray,
+        gesture_name: gestureName,
+        context: {
+          topic: document.title || 'learning',
+          timestamp: Date.now()
+        }
+      })
+    }).catch(err => console.log('Backend not available for gesture'));
+  }
+
+  applyFaceBlur(landmarks) {
+    if (!this.canvasElement || !this.videoElement) return;
+
+    const ctx = this.canvasElement.getContext('2d');
+    const video = this.videoElement;
+
+    if (video.videoWidth === 0 || video.videoHeight === 0) return;
+
+    this.canvasElement.width = video.videoWidth;
+    this.canvasElement.height = video.videoHeight;
+
+    ctx.drawImage(video, 0, 0);
+
+    let minX = 1, maxX = 0, minY = 1, maxY = 0;
+    for (const lm of landmarks) {
+      minX = Math.min(minX, lm.x);
+      maxX = Math.max(maxX, lm.x);
+      minY = Math.min(minY, lm.y);
+      maxY = Math.max(maxY, lm.y);
+    }
+
+    const padding = 0.15;
+    const padX = (maxX - minX) * padding;
+    const padY = (maxY - minY) * padding;
+
+    const x = Math.max(0, Math.floor((minX - padX) * video.videoWidth));
+    const y = Math.max(0, Math.floor((minY - padY) * video.videoHeight));
+    const w = Math.min(video.videoWidth, Math.floor((maxX + padX - minX + padX) * video.videoWidth));
+    const h = Math.min(video.videoHeight, Math.floor((maxY + padY - minY + padY) * video.videoHeight));
+
+    if (w > 10 && h > 10) {
+      const imageData = ctx.getImageData(x, y, w, h);
+      const data = imageData.data;
+      const pixelSize = 15;
+
+      for (let py = 0; py < h; py += pixelSize) {
+        for (let px = 0; px < w; px += pixelSize) {
+          const i = (py * w + px) * 4;
+          const r = data[i];
+          const g = data[i + 1];
+          const b = data[i + 2];
+
+          for (let dy = 0; dy < pixelSize && py + dy < h; dy++) {
+            for (let dx = 0; dx < pixelSize && px + dx < w; dx++) {
+              const ni = ((py + dy) * w + (px + dx)) * 4;
+              data[ni] = r;
+              data[ni + 1] = g;
+              data[ni + 2] = b;
+            }
+          }
+        }
+      }
+
+      ctx.putImageData(imageData, x, y);
+    }
+  }
+
+  drawHandLandmarks(landmarks) {
+    if (!this.overlayCanvasElement || !this.videoElement) return;
+
+    const canvas = this.overlayCanvasElement;
+    const video = this.videoElement;
+    
+    canvas.width = video.videoWidth;
+    canvas.height = video.videoHeight;
+
+    const ctx = canvas.getContext('2d');
+    ctx.clearRect(0, 0, canvas.width, canvas.height);
+
+    const connections = [
+      [0, 1], [1, 2], [2, 3], [3, 4],
+      [0, 5], [5, 6], [6, 7], [7, 8],
+      [0, 9], [9, 10], [10, 11], [11, 12],
+      [0, 13], [13, 14], [14, 15], [15, 16],
+      [0, 17], [17, 18], [18, 19], [19, 20],
+      [5, 9], [9, 13], [13, 17]
+    ];
+
+    ctx.strokeStyle = 'rgba(0, 255, 136, 0.8)';
+    ctx.lineWidth = 2;
+
+    for (const connection of connections) {
+      const start = landmarks[connection[0]];
+      const end = landmarks[connection[1]];
+      
+      ctx.beginPath();
+      ctx.moveTo(start.x * canvas.width, start.y * canvas.height);
+      ctx.lineTo(end.x * canvas.width, end.y * canvas.height);
+      ctx.stroke();
+    }
+
+    for (let i = 0; i < landmarks.length; i++) {
+      const landmark = landmarks[i];
+      const x = landmark.x * canvas.width;
+      const y = landmark.y * canvas.height;
+
+      ctx.beginPath();
+      ctx.arc(x, y, i % 4 === 0 ? 6 : 4, 0, 2 * Math.PI);
+      ctx.fillStyle = i % 4 === 0 ? '#00ff88' : '#ffffff';
+      ctx.fill();
+      ctx.strokeStyle = '#000000';
+      ctx.lineWidth = 1;
+      ctx.stroke();
+    }
+
+    const fingerCount = this.fingerCounter.count(landmarks);
+    ctx.fillStyle = '#ffffff';
+    ctx.font = 'bold 16px sans-serif';
+    ctx.fillText(`Fingers: ${fingerCount}`, 10, 30);
+  }
+
+  clearOverlay() {
+    if (!this.overlayCanvasElement) return;
+    
+    const ctx = this.overlayCanvasElement.getContext('2d');
+    ctx.clearRect(0, 0, this.overlayCanvasElement.width, this.overlayCanvasElement.height);
+  }
+
+  getLandmarks() {
+    return this.lastLandmarks;
+  }
+
+  getLandmarkHistory() {
+    return this.landmarkHistory;
+  }
+}
+
+class SwipeDetector {
+  constructor() {
+    this.swipeThreshold = 0.12;
+    this.minSwipeSpeed = 0.003;
+    this.minHistoryForSwipe = 10;
+    this.swipeStart = null;
+    this.isSwipeInProgress = false;
+    this.lastSwipeTime = 0;
+    this.swipeCooldown = 500;
+  }
+
+  detect(landmarks, fingerCount, history) {
+    if (!landmarks || landmarks.length < 21) return null;
+    if (history.length < this.minHistoryForSwipe) return null;
+
+    const now = Date.now();
+    if (now - this.lastSwipeTime < this.swipeCooldown) return null;
+
+    const wrist = landmarks[0];
+    const middleFingerMcp = landmarks[9];
+
+    const currentPos = {
+      x: middleFingerMcp.x,
+      y: middleFingerMcp.y,
+      z: middleFingerMcp.z || 0
+    };
+
+    if (!this.swipeStart && history.length >= 5) {
+      const recent = history.slice(-5);
+      const movementX = Math.abs(recent[recent.length - 1].landmarks[9][0] - recent[0].landmarks[9][0]);
+      const movementY = Math.abs(recent[recent.length - 1].landmarks[9][1] - recent[0].landmarks[9][1]);
+
+      if (movementX > this.swipeThreshold || movementY > this.swipeThreshold) {
+        this.swipeStart = { ...currentPos, time: now };
+        this.isSwipeInProgress = true;
+      }
+    }
+
+    if (this.isSwipeInProgress && this.swipeStart) {
+      const timeDelta = (now - this.swipeStart.time) / 1000;
+      const deltaX = currentPos.x - this.swipeStart.x;
+      const deltaY = currentPos.y - this.swipeStart.y;
+      
+      const distance = Math.sqrt(deltaX * deltaX + deltaY * deltaY);
+      const speed = distance / Math.max(timeDelta, 0.1);
+
+      const recentHistory = history.slice(-8);
+      const recentMovementX = Math.abs(
+        recentHistory[recentHistory.length - 1].landmarks[9][0] - 
+        recentHistory[0].landmarks[9][0]
+      );
+      const recentMovementY = Math.abs(
+        recentHistory[recentHistory.length - 1].landmarks[9][1] - 
+        recentHistory[0].landmarks[9][1]
+      );
+
+      if (recentMovementX < 0.008 && recentMovementY < 0.008 && distance > this.swipeThreshold) {
+        const direction = this.getSwipeDirection(deltaX, deltaY);
+        
+        this.swipeStart = null;
+        this.isSwipeInProgress = false;
+        this.lastSwipeTime = now;
+
+        return {
+          direction,
+          speed,
+          fingerCount,
+          startPosition: this.swipeStart ? {
+            x: this.swipeStart.x,
+            y: this.swipeStart.y
+          } : null,
+          endPosition: {
+            x: currentPos.x,
+            y: currentPos.y
+          }
+        };
+      }
+
+      if (timeDelta > 2) {
+        this.swipeStart = null;
+        this.isSwipeInProgress = false;
+      }
+    }
+
+    return null;
+  }
+
+  getSwipeDirection(dx, dy) {
+    const absDx = Math.abs(dx);
+    const absDy = Math.abs(dy);
+
+    if (absDx > absDy) {
+      return dx > 0 ? 'right' : 'left';
+    } else {
+      return dy > 0 ? 'down' : 'up';
+    }
+  }
+}
+
+class PinchDetector {
+  constructor() {
+    this.pinchThreshold = 0.08;
+    this.zoomInThreshold = 0.10;
+    this.zoomOutThreshold = 0.18;
+    this.lastPinchTime = 0;
+    this.pinchCooldown = 800;
+    this.isPinching = false;
+  }
+
+  detect(landmarks) {
+    if (!landmarks || landmarks.length < 21) return null;
+
+    const now = Date.now();
+    if (now - this.lastPinchTime < this.pinchCooldown) return null;
+
+    const thumbTip = landmarks[4];
+    const indexTip = landmarks[8];
+
+    const distance = Math.sqrt(
+      Math.pow(thumbTip.x - indexTip.x, 2) +
+      Math.pow(thumbTip.y - indexTip.y, 2) +
+      Math.pow((thumbTip.z || 0) - (indexTip.z || 0), 2)
+    );
+
+    let pinchType = null;
+
+    if (distance < this.pinchThreshold && !this.isPinching) {
+      pinchType = 'grab';
+      this.isPinching = true;
+      this.lastPinchTime = now;
+    } else if (distance > this.zoomOutThreshold && this.isPinching) {
+      pinchType = 'zoom_out';
+      this.isPinching = false;
+      this.lastPinchTime = now;
+    } else if (distance > this.zoomInThreshold && distance < this.zoomOutThreshold && this.isPinching) {
+      pinchType = 'zoom_in';
+      this.isPinching = false;
+      this.lastPinchTime = now;
+    } else if (distance > this.pinchThreshold * 2) {
+      this.isPinching = false;
+    }
+
+    if (pinchType) {
+      return {
+        type: pinchType,
+        distance,
+        thumbTip: [thumbTip.x, thumbTip.y, thumbTip.z || 0],
+        indexTip: [indexTip.x, indexTip.y, indexTip.z || 0]
+      };
+    }
+
+    return null;
+  }
+}
+
+class FingerCounter {
+  count(landmarks) {
+    if (!landmarks || landmarks.length < 21) return 0;
+
+    const fingerTips = [4, 8, 12, 16, 20];
+    const fingerBases = [3, 6, 10, 14, 18];
+
+    let extended = 0;
+
+    for (let i = 0; i < fingerTips.length; i++) {
+      const tip = landmarks[fingerTips[i]];
+      const base = landmarks[fingerBases[i]];
+
+      if (tip.y < base.y) {
+        extended++;
+      }
+    }
+
+    return extended;
+  }
+}
+
+export default MediaPipeProcessor;
+export { MediaPipeProcessor, SwipeDetector, PinchDetector, FingerCounter };