"use client" import { useState, useEffect } from "react" import { useUser } from "@stackframe/stack" import { Card, CardContent, CardHeader, CardTitle, CardDescription } from "@/components/ui/card" import { Button } from "@/components/ui/button" import { Badge } from "@/components/ui/badge" import { Textarea } from "@/components/ui/textarea" import { Input } from "@/components/ui/input" import { Progress } from "@/components/ui/progress" import { ShareMenu } from "@/components/share/share-menu" import { useOllama } from "@/contexts/ollama-context" import { BarChart3, Zap, Clock, DollarSign, Play, Loader2, Trophy, TrendingUp, TrendingDown, Save, Check, History, ChevronDown, ChevronUp, Rocket, Coins, Star, Server } from "lucide-react" import { ModelId } from "@/types/prompt" interface ModelPerformance { model: string name: string provider: string responseTime: number estimatedTokens: number estimatedCost: number qualityScore: number output: string isOllama?: boolean } const MODEL_PRICING: Record = { "gemini-2.0-flash": { input: 0.00001, output: 0.00004 }, "gpt-4o": { input: 0.0025, output: 0.01 }, "gpt-4o-mini": { input: 0.00015, output: 0.0006 }, "claude-3-5-sonnet-latest": { input: 0.003, output: 0.015 }, } const MODELS_TO_COMPARE: { id: ModelId; name: string; provider: string }[] = [ { id: "gemini-2.0-flash", name: "Gemini 2.0 Flash", provider: "Google" }, { id: "gpt-4o-mini", name: "GPT-4o Mini", provider: "OpenAI" }, { id: "gpt-4o", name: "GPT-4o", provider: "OpenAI" }, { id: "claude-3-5-sonnet", name: "Claude 3.5 Sonnet", provider: "Anthropic" }, ] interface SavedBenchmark { id: string name: string | null prompt: string results: { model: string; name: string; responseTime: number; tokens: number; cost: number }[] createdAt: string } export function PerformanceComparison() { const user = useUser() const { settings: ollamaSettings } = useOllama() const [prompt, setPrompt] = useState("") const [results, setResults] = useState([]) const [isRunning, setIsRunning] = useState(false) const [currentModel, setCurrentModel] = useState(null) // Save state const [saving, setSaving] = useState(false) const [saved, setSaved] = useState(false) const [saveName, setSaveName] = useState("") const [savedBenchmarks, setSavedBenchmarks] = useState([]) const [loadingHistory, setLoadingHistory] = useState(false) const [showHistory, setShowHistory] = useState(false) const [expandedId, setExpandedId] = useState(null) // Fetch saved benchmarks useEffect(() => { const fetchHistory = async () => { if (!user?.id) return setLoadingHistory(true) try { const res = await fetch(`/api/comparisons?userId=${user.id}&type=performance`) if (res.ok) { const data = await res.json() setSavedBenchmarks(data.comparisons || []) } } catch (err) { console.error("Failed to fetch history:", err) } finally { setLoadingHistory(false) } } fetchHistory() }, [user?.id]) const runComparison = async () => { if (!prompt.trim()) return setIsRunning(true) setResults([]) const newResults: ModelPerformance[] = [] // Combine cloud models with Ollama models const allModels = [ ...MODELS_TO_COMPARE, ...ollamaSettings.availableModels.map(name => ({ id: name, name: name, provider: "Ollama", isOllama: true })) ] for (const model of allModels) { setCurrentModel(model.name) const startTime = Date.now() try { const isOllama = 'isOllama' in model && model.isOllama let fullOutput = "" if (isOllama) { // Ollama: stream directly from browser const ollamaUrl = (ollamaSettings.apiUrl || "http://localhost:11434").replace(/\/+$/, "") const response = await fetch(`${ollamaUrl}/api/generate`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ model: model.id, prompt, stream: true }), }) const reader = response.body?.getReader() const decoder = new TextDecoder() if (reader) { while (true) { const { done, value } = await reader.read() if (done) break const chunk = decoder.decode(value, { stream: true }) const lines = chunk.split("\n").filter(l => l.trim()) for (const line of lines) { try { const json = JSON.parse(line) if (json.response) fullOutput += json.response } catch { /* skip */ } } } } } else { // Cloud models: stream via server /api/run const response = await fetch("/api/run", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ prompt, model: model.id }), }) const reader = response.body?.getReader() const decoder = new TextDecoder() if (reader) { while (true) { const { done, value } = await reader.read() if (done) break fullOutput += decoder.decode(value) } } } const endTime = Date.now() const responseTime = endTime - startTime // Estimate tokens (rough: ~4 chars per token) const inputTokens = Math.ceil(prompt.length / 4) const outputTokens = Math.ceil(fullOutput.length / 4) const totalTokens = inputTokens + outputTokens // Calculate cost (Ollama is free) const pricing = isOllama ? { input: 0, output: 0 } : (MODEL_PRICING[model.id] || { input: 0, output: 0 }) const cost = (inputTokens * pricing.input / 1000) + (outputTokens * pricing.output / 1000) newResults.push({ model: model.id, name: model.name, provider: model.provider, responseTime, estimatedTokens: totalTokens, estimatedCost: cost, qualityScore: 0, output: fullOutput, isOllama: isOllama, }) setResults([...newResults]) } catch (error) { console.error(`Error with ${model.name}:`, error) } } setCurrentModel(null) setIsRunning(false) } // Rate quality of a model's output const rateQuality = (modelId: string, score: number) => { setResults(prev => prev.map(r => r.model === modelId ? { ...r, qualityScore: score } : r )) } // Find best performers const fastestModel = results.length > 0 ? results.reduce((a, b) => a.responseTime < b.responseTime ? a : b) : null const cheapestModel = results.length > 0 ? results.reduce((a, b) => a.estimatedCost < b.estimatedCost ? a : b) : null const mostEfficient = results.length > 0 ? results.reduce((a, b) => (a.estimatedTokens / a.responseTime) > (b.estimatedTokens / b.responseTime) ? a : b) : null // Save benchmark const handleSave = async () => { if (results.length === 0) return setSaving(true) try { const res = await fetch('/api/comparisons', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ userId: user?.id, name: saveName || null, type: 'performance', prompt, results: results.map(r => ({ model: r.model, name: r.name, responseTime: r.responseTime, tokens: r.estimatedTokens, cost: r.estimatedCost })), winner: fastestModel?.model }) }) if (res.ok) { const newBench = await res.json() setSavedBenchmarks(prev => [newBench, ...prev]) setSaved(true) setSaveName("") setTimeout(() => setSaved(false), 3000) } } catch (err) { console.error("Failed to save:", err) } finally { setSaving(false) } } return (
{/* Hero */}

Performance Comparison

Compare speed, token usage, and cost across AI models

{/* Input */} Test Prompt Enter a prompt to benchmark across all models