Add main.py desktop GUI

main.py

@@ -0,0 +1,260 @@
+"""
+Buddy — Desktop Push-to-Talk AI Assistant
+
+Hold F9 for å snakke. Assistenten ser skjermbildet ditt,
+transkriberer talen, svarer via multimodal VLM,
+og snakker tilbake.
+
+Krav:
+    pip install -r requirements.txt
+
+Bruk:
+    python main.py
+
+Hotkeys:
+    F9      — Hold for å snakke, slipp for å sende
+    Esc     — Avslutt
+"""
+import tkinter as tk
+from tkinter import ttk, scrolledtext
+from PIL import ImageTk, Image
+import threading
+import queue
+import time
+import sys
+
+from pynput import keyboard
+
+from audio_io import PushToTalkRecorder, TTSEngine
+from screen_capture import capture_primary_monitor
+from vision_llm import MultimodalAssistant
+
+
+class BuddyApp:
+    def __init__(self):
+        self.root = tk.Tk()
+        self.root.title("Buddy — Push-to-Talk AI Assistant")
+        self.root.geometry("900x750")
+        self.root.configure(bg="#1e1e2e")
+
+        # --- Styling ---
+        self.style = ttk.Style()
+        self.style.theme_use("clam")
+        self.style.configure("TFrame", background="#1e1e2e")
+        self.style.configure("TLabel", background="#1e1e2e", foreground="#cdd6f4", font=("Inter", 11))
+        self.style.configure("Status.TLabel", background="#1e1e2e", foreground="#a6e3a1", font=("Inter", 12, "bold"))
+        self.style.configure("Recording.TLabel", background="#1e1e2e", foreground="#f38ba8", font=("Inter", 12, "bold"))
+
+        # --- State ---
+        self.recorder = PushToTalkRecorder()
+        self.tts = TTSEngine(voice="nb-NO-FinnNeural")
+        self.assistant = None
+        self.model_loaded = False
+
+        self.msg_queue = queue.Queue()
+        self.history = []
+
+        self._build_ui()
+        self._start_keyboard_listener()
+        self._process_queue()
+
+        # Load model in background
+        threading.Thread(target=self._load_model, daemon=True).start()
+
+    def _build_ui(self):
+        # Header
+        header = ttk.Frame(self.root)
+        header.pack(fill="x", padx=20, pady=(20, 10))
+
+        ttk.Label(header, text="Buddy", font=("Inter", 24, "bold"), foreground="#89b4fa").pack(side="left")
+        self.status_label = ttk.Label(header, text="Laster modeller...", style="Status.TLabel")
+        self.status_label.pack(side="right")
+
+        # Screenshot preview
+        self.screenshot_label = ttk.Label(self.root, text="Skjermbilde vises her")
+        self.screenshot_label.pack(padx=20, pady=10)
+
+        # Chat log
+        chat_frame = ttk.Frame(self.root)
+        chat_frame.pack(fill="both", expand=True, padx=20, pady=10)
+
+        self.chat_text = scrolledtext.ScrolledText(
+            chat_frame,
+            wrap=tk.WORD,
+            bg="#181825",
+            fg="#cdd6f4",
+            insertbackground="#cdd6f4",
+            font=("Inter", 11),
+            borderwidth=0,
+            padx=10,
+            pady=10,
+        )
+        self.chat_text.pack(fill="both", expand=True)
+        self.chat_text.config(state=tk.DISABLED)
+
+        # Controls
+        controls = ttk.Frame(self.root)
+        controls.pack(fill="x", padx=20, pady=(10, 20))
+
+        self.ptt_btn = tk.Button(
+            controls,
+            text="Hold F9 for å snakke (eller trykk her)",
+            font=("Inter", 12, "bold"),
+            bg="#89b4fa",
+            fg="#1e1e2e",
+            activebackground="#b4befe",
+            relief=tk.FLAT,
+            padx=20,
+            pady=10,
+            cursor="hand2",
+        )
+        self.ptt_btn.pack(side="left", fill="x", expand=True)
+
+        # Mouse push-to-talk
+        self.ptt_btn.bind("<ButtonPress-1>", lambda e: self._start_recording())
+        self.ptt_btn.bind("<ButtonRelease-1>", lambda e: self._stop_recording())
+
+        # Hint
+        hint = ttk.Label(
+            self.root,
+            text="F9 = Push-to-Talk  |  Esc = Avslutt  |  Assistenten ser automatisk skjermbildet ditt",
+            font=("Inter", 9),
+            foreground="#6c7086",
+        )
+        hint.pack(pady=(0, 15))
+
+    def _append_chat(self, sender, text):
+        self.chat_text.config(state=tk.NORMAL)
+        if sender == "Du":
+            line = "Du: " + text + "\n\n"
+            self.chat_text.insert(tk.END, line)
+        else:
+            line = "Buddy: " + text + "\n\n"
+            self.chat_text.insert(tk.END, line)
+        self.chat_text.config(state=tk.DISABLED)
+        self.chat_text.see(tk.END)
+
+    def _update_screenshot(self):
+        try:
+            img = capture_primary_monitor()
+            img.thumbnail((800, 450))
+            self.tk_screenshot = ImageTk.PhotoImage(img)
+            self.screenshot_label.config(image=self.tk_screenshot, text="")
+        except Exception as e:
+            self.screenshot_label.config(text="Kunne ikke ta skjermbilde: " + str(e))
+
+    def _load_model(self):
+        try:
+            self.assistant = MultimodalAssistant()
+            self.model_loaded = True
+            self.msg_queue.put(("status", "Klar! Hold F9 for å snakke"))
+        except Exception as e:
+            self.msg_queue.put(("status", "Feil ved lasting: " + str(e)))
+            print("[init] Model load error: " + str(e))
+
+    def _start_recording(self):
+        if not self.model_loaded:
+            self.msg_queue.put(("status", "Vent, modell laster ennaa..."))
+            return
+        if self.recorder.is_recording():
+            return
+
+        self.recorder.start()
+        self.msg_queue.put(("status_recording", "Lytter... slipp for aa sende"))
+        self._update_screenshot()
+
+    def _stop_recording(self):
+        if not self.recorder.is_recording():
+            return
+
+        audio_bytes = self.recorder.stop()
+        self.msg_queue.put(("status", "Transkriberer..."))
+
+        if not audio_bytes:
+            self.msg_queue.put(("status", "Ingen lyd fanget opp"))
+            return
+
+        # Process in background thread
+        threading.Thread(
+            target=self._process_turn,
+            args=(audio_bytes,),
+            daemon=True,
+        ).start()
+
+    def _process_turn(self, audio_bytes):
+        try:
+            # 1. STT
+            transcript = self.assistant.transcribe_audio(audio_bytes)
+            self.msg_queue.put(("chat_user", transcript))
+
+            # 2. Capture fresh screenshot
+            self.msg_queue.put(("status", "Ser paa skjermbildet..."))
+            screenshot = capture_primary_monitor()
+            self.msg_queue.put(("screenshot", None))
+
+            # 3. VLM
+            self.msg_queue.put(("status", "Tenker..."))
+            response = self.assistant.ask_with_image(screenshot, transcript)
+
+            # 4. Update UI + TTS
+            self.msg_queue.put(("chat_buddy", response))
+            self.msg_queue.put(("status", "Klar!"))
+            self.tts.speak(response, blocking=False)
+
+        except Exception as e:
+            self.msg_queue.put(("status", "Feil: " + str(e)))
+            print("[turn] Error: " + str(e))
+
+    def _start_keyboard_listener(self):
+        def on_press(key):
+            if key == keyboard.Key.f9 and self.model_loaded and not self.recorder.is_recording():
+                self.msg_queue.put(("start_recording", None))
+            if key == keyboard.Key.esc:
+                self.msg_queue.put(("quit", None))
+
+        def on_release(key):
+            if key == keyboard.Key.f9 and self.recorder.is_recording():
+                self.msg_queue.put(("stop_recording", None))
+
+        self.listener = keyboard.Listener(on_press=on_press, on_release=on_release)
+        self.listener.start()
+
+    def _process_queue(self):
+        try:
+            while True:
+                kind, payload = self.msg_queue.get_nowait()
+
+                if kind == "status":
+                    self.status_label.config(text=payload, style="Status.TLabel")
+                elif kind == "status_recording":
+                    self.status_label.config(text=payload, style="Recording.TLabel")
+                elif kind == "chat_user":
+                    self._append_chat("Du", payload)
+                elif kind == "chat_buddy":
+                    self._append_chat("Buddy", payload)
+                elif kind == "screenshot":
+                    self._update_screenshot()
+                elif kind == "start_recording":
+                    self._start_recording()
+                elif kind == "stop_recording":
+                    self._stop_recording()
+                elif kind == "quit":
+                    self._quit()
+
+        except queue.Empty:
+            pass
+
+        self.root.after(50, self._process_queue)
+
+    def _quit(self):
+        self.listener.stop()
+        self.root.destroy()
+        sys.exit(0)
+
+    def run(self):
+        self.root.mainloop()
+
+
+if __name__ == "__main__":
+    app = BuddyApp()
+    app.run()