Scores and multi-provider

app.py

@@ -31,7 +31,10 @@ from io_utils import (
     export_tsv, export_conllu, export_jsonl_finetune,
 )
 from moe import aggregate
-from provider import OpenRouterClient, CURATED_MODELS, test_connection_sync
+from provider import (
+    LLMClient, PROVIDERS, BASE_URLS,
+    CURATED_MODELS_BY_PROVIDER, test_connection_sync,
+)
 from tutorial import EXERCISES, prefill
 
 
@@ -59,7 +62,8 @@ SESSION: dict[str, Any] = {
     "language": "",
     "system_prompt": DEFAULT_SYSTEM_PROMPT,
     "user_template": DEFAULT_FEW_SHOT,
-    "models": ["openai/gpt-oss-20b:free"],
+    "provider": "openrouter",
+    "models": list(CURATED_MODELS_BY_PROVIDER["openrouter"][:1]),
     "priority": [],
     "temperature": 0.0,
     "n_icl": 5,
@@ -116,7 +120,10 @@ def _public_state() -> dict:
         },
         "sentences": sess["sentences"],
         "presets": [{"key": k, "label": label} for k, label in list_presets()],
-        "curated_models": CURATED_MODELS,
+        "provider": sess["provider"],
+        "providers": list(PROVIDERS),
+        "curated_models": CURATED_MODELS_BY_PROVIDER.get(sess["provider"], []),
+        "curated_models_by_provider": CURATED_MODELS_BY_PROVIDER,
         "aggregators": AGGREGATORS,
         "exercises": [
             {"idx": i, "title": ex.title, "summary": ex.summary, "language": ex.language_code, "models": ex.models}
@@ -149,6 +156,7 @@ class LoadExerciseReq(BaseModel):
 
 
 class SettingsReq(BaseModel):
+    provider: Optional[str] = None
     models: Optional[list[str]] = None
     priority: Optional[list[str]] = None
     temperature: Optional[float] = None
@@ -168,7 +176,8 @@ class AnnotateReq(BaseModel):
 
 class TestKeyReq(BaseModel):
     api_key: str
-    model: Optional[str] = "openai/gpt-4o-mini"
+    provider: Optional[str] = "openrouter"
+    model: Optional[str] = None
 
 
 # ---------------------------------------------------------------------------
@@ -223,8 +232,20 @@ def set_task_schema(req: TaskSchemaReq):
 
 @app.post("/api/settings")
 def set_settings(req: SettingsReq):
+    if req.provider is not None:
+        if req.provider not in PROVIDERS:
+            raise HTTPException(400, f"Unknown provider {req.provider!r}; expected one of {list(PROVIDERS)}")
+        if req.provider != SESSION["provider"]:
+            SESSION["provider"] = req.provider
+            # reset to the new provider's first curated model (if any), to avoid orphan slugs
+            curated = CURATED_MODELS_BY_PROVIDER.get(req.provider) or []
+            SESSION["models"] = list(curated[:1])
     if req.models is not None:
-        SESSION["models"] = list(req.models)
+        models = list(req.models)
+        # Non-OpenRouter providers don't support MoE → keep a single model
+        if SESSION["provider"] != "openrouter" and len(models) > 1:
+            models = models[:1]
+        SESSION["models"] = models
     if req.priority is not None:
         SESSION["priority"] = list(req.priority)
     if req.temperature is not None:
@@ -242,7 +263,8 @@ def set_settings(req: SettingsReq):
 
 @app.post("/api/settings/test_key")
 def test_key(req: TestKeyReq):
-    ok, msg = test_connection_sync(req.api_key, model=req.model or "openai/gpt-4o-mini")
+    provider = req.provider or "openrouter"
+    ok, msg = test_connection_sync(req.api_key, provider=provider, model=req.model)
     return {"ok": ok, "message": msg}
 
 
@@ -308,7 +330,8 @@ def reset_all():
     """Wipe everything except the API key (which lives client-side)."""
     SESSION["schema"] = _default_schema().to_dict()
     SESSION["language"] = ""
-    SESSION["models"] = ["openai/gpt-oss-20b:free"]
+    SESSION["provider"] = "openrouter"
+    SESSION["models"] = list(CURATED_MODELS_BY_PROVIDER["openrouter"][:1])
     SESSION["priority"] = []
     SESSION["temperature"] = 0.0
     SESSION["n_icl"] = 5
@@ -435,7 +458,7 @@ def icl_download():
 
 # --- annotation ------------------------------------------------------------
 
-async def _annotate_sentence(sent: dict, client: OpenRouterClient,
+async def _annotate_sentence(sent: dict, client: LLMClient,
                               schema: AnnotationSchema, sys_prompt: str,
                               user_template: str, language: str,
                               pool: ICLPool, n_icl: int, temperature: float,
@@ -495,15 +518,25 @@ async def _annotate_sentence(sent: dict, client: OpenRouterClient,
 
 
 @app.post("/api/annotate")
-async def annotate(req: AnnotateReq, x_openrouter_key: Optional[str] = Header(default=None)):
+async def annotate(
+    req: AnnotateReq,
+    x_api_key: Optional[str] = Header(default=None),
+    x_openrouter_key: Optional[str] = Header(default=None),  # back-compat
+    x_llm_provider: Optional[str] = Header(default=None),
+):
     sess = SESSION
-    api_key = _resolve_key(x_openrouter_key)
+    provider = (x_llm_provider or sess["provider"]).strip()
+    if provider not in PROVIDERS:
+        raise HTTPException(400, f"Unknown provider {provider!r}")
+    api_key = _resolve_key(x_api_key or x_openrouter_key)
     if not api_key:
-        raise HTTPException(400, "Set your OpenRouter API key first.")
+        raise HTTPException(400, f"Set your {provider} API key first.")
     if not sess["models"]:
         raise HTTPException(400, "Select at least one model.")
+    if provider != "openrouter" and len(sess["models"]) > 1:
+        raise HTTPException(400, f"MoE (multiple models) is only supported on OpenRouter. Pick one model for {provider}.")
     schema_obj = schema_from_dict(sess["schema"])
-    client = OpenRouterClient(api_key=api_key)
+    client = LLMClient(provider=provider, api_key=api_key)
     pool: ICLPool = sess["icl_pool"]
     sents = sess["sentences"]
     target_idxs = req.sentence_idxs if req.sentence_idxs is not None else list(range(len(sents)))
@@ -521,12 +554,20 @@ async def annotate(req: AnnotateReq, x_openrouter_key: Optional[str] = Header(de
 
 
 @app.post("/api/annotate/token")
-async def annotate_one_token(payload: dict, x_openrouter_key: Optional[str] = Header(default=None)):
+async def annotate_one_token(
+    payload: dict,
+    x_api_key: Optional[str] = Header(default=None),
+    x_openrouter_key: Optional[str] = Header(default=None),
+    x_llm_provider: Optional[str] = Header(default=None),
+):
     """Re-ask a specific model for a specific token. payload = {sent: int, tok: int, model: str}"""
     sess = SESSION
-    api_key = _resolve_key(x_openrouter_key)
+    provider = (x_llm_provider or sess["provider"]).strip()
+    if provider not in PROVIDERS:
+        raise HTTPException(400, f"Unknown provider {provider!r}")
+    api_key = _resolve_key(x_api_key or x_openrouter_key)
     if not api_key:
-        raise HTTPException(400, "Set your OpenRouter API key first.")
+        raise HTTPException(400, f"Set your {provider} API key first.")
     idx = int(payload["sent"])
     tidx = int(payload["tok"])
     model = str(payload["model"])
@@ -540,7 +581,7 @@ async def annotate_one_token(payload: dict, x_openrouter_key: Optional[str] = He
         language=sess["language"] or sent["language"], sentence_id=sent["id"],
         few_shot_examples=examples,
     ) + f"\n\nFocus especially on token index {tidx} (surface={tokens[tidx]!r}). Return JSON for all tokens; preserve the order."
-    client = OpenRouterClient(api_key=api_key)
+    client = LLMClient(provider=provider, api_key=api_key)
     result = await client.annotate_one(
         system=sess["system_prompt"], user=rendered_user,
         schema=schema, model=model, temperature=float(sess["temperature"]),

provider.py

@@ -1,8 +1,13 @@
-"""OpenRouter async client for structured-output annotation.
+"""LLM provider client for structured-output annotation.
 
-One gateway, many models. Mirrors the OpenAI chat-completions interface so
-JSON-Schema response_format works uniformly across Claude, GPT, Mistral,
-Llama, Qwen, etc.
+Supports three OpenAI-compatible providers via parametric base URL:
+- openrouter : https://openrouter.ai/api/v1   (MoE: many models behind one key)
+- mistral    : https://api.mistral.ai/v1
+- openai     : https://api.openai.com/v1
+
+All three accept the same OpenAI Chat Completions request shape, including
+`response_format` (json_schema strict on OpenAI; json_object on Mistral; varies
+on OpenRouter — we auto-fall back if the strict mode is rejected).
 """
 from __future__ import annotations
 
@@ -10,26 +15,50 @@ import asyncio
 import json
 import time
 from dataclasses import dataclass
-from typing import Any, Optional
+from typing import Optional
 
 import httpx
 
 from schemas import AnnotationSchema, to_json_schema, validate as schema_validate
 from prompts import VALIDATION_RETRY
 
-OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions"
 DEFAULT_TIMEOUT = 60.0
 
-CURATED_MODELS = [
-    "openai/gpt-oss-20b:free",
-    "google/gemma-4-26b-a4b-it:free",
-    "meta-llama/llama-3.3-70b-instruct:free",
-    "qwen/qwen3-next-80b-a3b-instruct:free",
-    "deepseek/deepseek-v4-flash:free",
-    "mistralai/mistral-nemo",
-    "mistralai/mistral-small-24b-instruct-2501",
-    "mistralai/ministral-3b-2512",
-]
+PROVIDERS = ("openrouter", "mistral", "openai")
+
+BASE_URLS = {
+    "openrouter": "https://openrouter.ai/api/v1",
+    "mistral":    "https://api.mistral.ai/v1",
+    "openai":     "https://api.openai.com/v1",
+}
+
+CURATED_MODELS_BY_PROVIDER: dict[str, list[str]] = {
+    "openrouter": [
+        "openai/gpt-oss-20b:free",
+        "google/gemma-4-26b-a4b-it:free",
+        "meta-llama/llama-3.3-70b-instruct:free",
+        "qwen/qwen3-next-80b-a3b-instruct:free",
+        "deepseek/deepseek-v4-flash:free",
+        "mistralai/mistral-nemo",
+        "mistralai/mistral-small-24b-instruct-2501",
+        "mistralai/ministral-3b-2512",
+    ],
+    "mistral": [
+        "mistral-small-2603",
+        "mistral-large-2512",
+        "ministral-8b-2512",
+        "ministral-3b-2512",
+    ],
+    "openai": [
+        "gpt-5-mini-2025-08-07",
+        "gpt-5-nano-2025-08-07",
+        "gpt-5-2025-08-07",
+        "gpt-4o-mini-2024-07-18",
+    ],
+}
+
+# Back-compat alias used by other modules
+CURATED_MODELS = CURATED_MODELS_BY_PROVIDER["openrouter"]
 
 
 @dataclass
@@ -42,15 +71,26 @@ class ModelResult:
     raw: str = ""
 
 
-class OpenRouterClient:
-    def __init__(self, api_key: str, http_referer: str = "https://lrec2026-llm-annotator.local", app_name: str = "LREC2026 LLM-as-Annotator"):
+def _build_headers(provider: str, api_key: str) -> dict:
+    h = {
+        "Authorization": f"Bearer {api_key}",
+        "Content-Type": "application/json",
+    }
+    if provider == "openrouter":
+        h["HTTP-Referer"] = "https://lrec2026-llm-annotator.local"
+        h["X-Title"] = "LREC2026 LLM-as-Annotator"
+    return h
+
+
+class LLMClient:
+    def __init__(self, provider: str, api_key: str):
+        if provider not in BASE_URLS:
+            raise ValueError(f"Unknown provider {provider!r}; expected one of {PROVIDERS}")
+        self.provider = provider
         self.api_key = api_key
-        self.headers = {
-            "Authorization": f"Bearer {api_key}",
-            "HTTP-Referer": http_referer,
-            "X-Title": app_name,
-            "Content-Type": "application/json",
-        }
+        self.base_url = BASE_URLS[provider]
+        self.endpoint = self.base_url + "/chat/completions"
+        self.headers = _build_headers(provider, api_key)
 
     async def annotate_one(
         self,
@@ -71,7 +111,6 @@ class OpenRouterClient:
                 raw_text = await self._call(client, msgs, json_schema, model, temperature)
                 ann, err = self._parse_and_validate(raw_text, schema)
                 if err:
-                    # one retry with validator error appended
                     retry_msg = VALIDATION_RETRY + f"\n\nValidator errors:\n{err}\n\nPrevious response:\n{raw_text}"
                     msgs.append({"role": "assistant", "content": raw_text})
                     msgs.append({"role": "user", "content": retry_msg})
@@ -102,28 +141,32 @@ class OpenRouterClient:
         return await asyncio.gather(*coros)
 
     async def _call(self, client: httpx.AsyncClient, msgs: list[dict], json_schema: dict, model: str, temperature: float) -> str:
-        payload = {
-            "model": model,
-            "messages": msgs,
-            "temperature": temperature,
-            "response_format": {
-                "type": "json_schema",
-                "json_schema": {"name": "annotation", "strict": True, "schema": json_schema},
-            },
-        }
-        resp = await client.post(OPENROUTER_URL, headers=self.headers, json=payload)
+        # Strict json_schema works on OpenAI and most OpenRouter models. For Mistral and
+        # for some open-source models routed via OpenRouter, fall back to json_object.
+        if self.provider == "mistral":
+            payload = {
+                "model": model, "messages": msgs, "temperature": temperature,
+                "response_format": {"type": "json_object"},
+            }
+        else:
+            payload = {
+                "model": model, "messages": msgs, "temperature": temperature,
+                "response_format": {
+                    "type": "json_schema",
+                    "json_schema": {"name": "annotation", "strict": True, "schema": json_schema},
+                },
+            }
+        resp = await client.post(self.endpoint, headers=self.headers, json=payload)
         if resp.status_code >= 400:
-            # Some models don't honor strict json_schema; fall back to json_object then trust prompt instructions.
             payload["response_format"] = {"type": "json_object"}
-            resp = await client.post(OPENROUTER_URL, headers=self.headers, json=payload)
+            resp = await client.post(self.endpoint, headers=self.headers, json=payload)
         resp.raise_for_status()
         data = resp.json()
         return data["choices"][0]["message"]["content"] or ""
 
     @staticmethod
     def _parse_and_validate(raw_text: str, schema: AnnotationSchema) -> tuple[Optional[dict], str]:
-        text = raw_text.strip()
-        # tolerate ```json ... ``` fences
+        text = (raw_text or "").strip()
         if text.startswith("```"):
             text = text.strip("`")
             if text.lower().startswith("json"):
@@ -139,15 +182,20 @@ class OpenRouterClient:
         return ann, ""
 
 
-def test_connection_sync(api_key: str, model: str = "openai/gpt-oss-20b:free") -> tuple[bool, str]:
-    """Quick blocking test for the Models tab "Test connection" button."""
+def test_connection_sync(api_key: str, provider: str = "openrouter", model: Optional[str] = None) -> tuple[bool, str]:
+    """Quick blocking test for the 'Test' button in the key modal."""
+    if provider not in BASE_URLS:
+        return False, f"Unknown provider {provider!r}"
+    if not model:
+        models = CURATED_MODELS_BY_PROVIDER.get(provider) or []
+        model = models[0] if models else None
+    if not model:
+        return False, "No model configured for this provider."
+    url = BASE_URLS[provider] + "/chat/completions"
     try:
         resp = httpx.post(
-            OPENROUTER_URL,
-            headers={
-                "Authorization": f"Bearer {api_key}",
-                "Content-Type": "application/json",
-            },
+            url,
+            headers=_build_headers(provider, api_key),
             json={
                 "model": model,
                 "messages": [{"role": "user", "content": "Reply with the single word: OK"}],
@@ -159,6 +207,12 @@ def test_connection_sync(api_key: str, model: str = "openai/gpt-oss-20b:free") -
         if resp.status_code >= 400:
             return False, f"HTTP {resp.status_code}: {resp.text[:200]}"
         content = resp.json()["choices"][0]["message"]["content"]
-        return True, f"Connected. Reply: {content!r}"
+        return True, f"Connected ({provider} / {model}). Reply: {content!r}"
     except Exception as e:
         return False, str(e)
+
+
+# Back-compat shim — old callers used OpenRouterClient(api_key=...).
+class OpenRouterClient(LLMClient):
+    def __init__(self, api_key: str, **kwargs):
+        super().__init__(provider="openrouter", api_key=api_key)

static/app.js

@@ -16,7 +16,8 @@ function annotator() {
     guideDismissed: false,
     moeBannerDismissed: false,
     moeHintDismissed: false,
-    localKey: '',  // client-side OpenRouter key; never sent to /api/state, never persisted on server
+    // Per-provider client-side keys; persisted in sessionStorage only
+    localKeys: { openrouter: '', mistral: '', openai: '' },
 
     state: {
       schema: null,
@@ -26,6 +27,9 @@ function annotator() {
       system_prompt: '',
       user_template: '',
       has_env_key: false,
+      provider: 'openrouter',
+      providers: ['openrouter', 'mistral', 'openai'],
+      curated_models_by_provider: {},
       models: [],
       priority: [],
       temperature: 0,
@@ -80,15 +84,25 @@ function annotator() {
     get totalDisagreements() {
       return this.state.sentences.reduce((a, s) => a + (s.n_disagreements || 0), 0);
     },
+    // ----------- key helpers (per-provider) -----------
+    get localKey() { return this.localKeys[this.state.provider] || ''; },
+    setLocalKey(value) {
+      const p = this.state.provider;
+      this.localKeys = { ...this.localKeys, [p]: value };
+      try { sessionStorage.setItem('llm_keys', JSON.stringify(this.localKeys)); } catch (e) {}
+    },
     get hasKey() {
-      return !!this.localKey || !!this.state.has_env_key;
+      // env key is OpenRouter-only on the server side
+      const envOk = this.state.provider === 'openrouter' && !!this.state.has_env_key;
+      return !!this.localKey || envOk;
     },
     get canRun() {
       return this.hasKey && this.state.models.length > 0 && this.state.sentences.length > 0;
     },
     keyHeaders() {
-      // Send the client-side key as a header. Server never stores it.
-      return this.localKey ? { 'X-OpenRouter-Key': this.localKey } : {};
+      const h = { 'X-LLM-Provider': this.state.provider };
+      if (this.localKey) h['X-API-Key'] = this.localKey;
+      return h;
     },
 
     // ----------- init -----------
@@ -96,7 +110,17 @@ function annotator() {
       this.guideDismissed = localStorage.getItem('guideDismissed') === '1';
       this.moeBannerDismissed = localStorage.getItem('moeBannerDismissed') === '1';
       this.moeHintDismissed = localStorage.getItem('moeHintDismissed') === '1';
-      this.localKey = sessionStorage.getItem('openrouter_key') || '';
+      // Load per-provider keys; migrate legacy single-key key if present
+      try {
+        const raw = sessionStorage.getItem('llm_keys');
+        if (raw) this.localKeys = { openrouter: '', mistral: '', openai: '', ...JSON.parse(raw) };
+        const legacy = sessionStorage.getItem('openrouter_key');
+        if (legacy && !this.localKeys.openrouter) {
+          this.localKeys = { ...this.localKeys, openrouter: legacy };
+          sessionStorage.setItem('llm_keys', JSON.stringify(this.localKeys));
+          sessionStorage.removeItem('openrouter_key');
+        }
+      } catch (e) {}
       await this.refresh();
       try {
         const r = await fetch('/api/cheatsheet');
@@ -297,6 +321,51 @@ function annotator() {
       return parts[parts.length - 1];
     },
 
+    // Per-model accuracy on a single sentence, computed vs the current consensus
+    // (i.e., what the user sees after MoE aggregation + their corrections).
+    // Skips confidence/comment (same `min`/`priority` aggregators ignored for disagreements).
+    modelAccuracy(sent) {
+      if (!sent || sent.status !== 'done') return [];
+      const perModel = sent.per_model || {};
+      const modelNames = Object.keys(perModel);
+      if (modelNames.length === 0) return [];
+      const quiet = new Set(['min', 'priority']);
+      const fields = (this.state.schema?.fields || []).filter(f => !quiet.has(f.aggregator));
+      const out = [];
+      for (const m of modelNames) {
+        const tokens = perModel[m].tokens || [];
+        let total = 0, correct = 0;
+        const n = Math.min(tokens.length, sent.tokens.length);
+        for (let i = 0; i < n; i++) {
+          const got = tokens[i] || {};
+          const ref = sent.tokens[i] || {};
+          for (const f of fields) {
+            if (f.type === 'object') {
+              for (const sub of (f.subfields || [])) {
+                const a = (got[f.name] || {})[sub.name] ?? null;
+                const b = (ref[f.name] || {})[sub.name] ?? null;
+                total++;
+                if (a === b) correct++;
+              }
+            } else {
+              const a = got[f.name] ?? null;
+              const b = ref[f.name] ?? null;
+              total++;
+              if (a === b) correct++;
+            }
+          }
+        }
+        out.push({ model: m, pct: total > 0 ? Math.round(100 * correct / total) : 0, correct, total });
+      }
+      return out.sort((a, b) => b.pct - a.pct);
+    },
+
+    accuracyClass(pct) {
+      if (pct >= 90) return 'accuracy-pill-high';
+      if (pct >= 70) return 'accuracy-pill-mid';
+      return 'accuracy-pill-low';
+    },
+
     modelTokenSummary(ann, tidx) {
       const t = (ann.tokens || [])[tidx] || {};
       const enums = (this.state.schema?.fields || []).filter(f => f.type === 'enum' && f.name !== 'confidence');
@@ -335,6 +404,12 @@ function annotator() {
       this.applyState(await r.json());
     },
 
+    async setProvider(p) {
+      if (!this.state.providers.includes(p)) return;
+      await this.saveSettings({ provider: p });
+      this.toast(`Provider: ${p}. Models reset to its defaults.`, 'ok');
+    },
+
     saveKey() {
       const k = (this.keyEditor.value || '').trim();
       if (!k) {
@@ -345,22 +420,20 @@ function annotator() {
         }
         return;
       }
-      this.localKey = k;
-      try { sessionStorage.setItem('openrouter_key', k); } catch (e) {}
+      this.setLocalKey(k);
       this.keyEditor.value = '';
       this.keyEditor.result = '';
       this.keyEditor.ok = false;
-      this.toast(`✓ Key saved in this tab (${k.length} chars). Pill above should turn green.`, 'ok');
+      this.toast(`✓ ${this.state.provider} key saved in this tab (${k.length} chars). Pill above should turn green.`, 'ok');
       this.closeModal();
     },
 
     clearKey() {
-      this.localKey = '';
-      try { sessionStorage.removeItem('openrouter_key'); } catch (e) {}
+      this.setLocalKey('');
       this.keyEditor.value = '';
       this.keyEditor.result = '';
       this.keyEditor.ok = false;
-      this.toast('Key cleared from this tab.', 'ok');
+      this.toast(`${this.state.provider} key cleared from this tab.`, 'ok');
     },
 
     async testKey(autoSaveOnSuccess = false) {
@@ -369,15 +442,14 @@ function annotator() {
       this.keyEditor.testing = true;
       this.keyEditor.result = '';
       try {
-        const r = await fetch('/api/settings/test_key', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ api_key: k }) });
+        const r = await fetch('/api/settings/test_key', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ api_key: k, provider: this.state.provider }) });
         const j = await r.json();
         this.keyEditor.ok = j.ok;
         this.keyEditor.result = (j.ok ? '✓ ' : '✗ ') + j.message;
         if (j.ok && autoSaveOnSuccess) {
-          this.localKey = k;
-          try { sessionStorage.setItem('openrouter_key', k); } catch (e) {}
+          this.setLocalKey(k);
           this.keyEditor.value = '';
-          this.toast(`✓ Key tested & saved (${k.length} chars).`, 'ok');
+          this.toast(`✓ ${this.state.provider} key tested & saved (${k.length} chars).`, 'ok');
         }
       } catch (e) {
         this.keyEditor.ok = false;

static/index.html

@@ -24,7 +24,7 @@
       }
     }
   </script>
-  <link rel="stylesheet" href="/static/styles.css?v=20260516b">
+  <link rel="stylesheet" href="/static/styles.css?v=20260516c">
   <script defer src="https://cdn.jsdelivr.net/npm/alpinejs@3.13.10/dist/cdn.min.js"></script>
 </head>
 <body class="font-sans bg-ink-50 text-ink-900 min-h-screen" x-data="annotator()" x-init="init()" x-cloak>
@@ -53,19 +53,19 @@
 
       <!-- Models pill -->
       <button @click="modal='models'" class="pill"
-              :class="state.models.length >= 2 ? 'pill-moe' : ''"
+              :class="(state.provider === 'openrouter' && state.models.length >= 2) ? 'pill-moe' : ''"
               :title="state.models.join(', ')">
-        <span class="pill-icon" x-text="state.models.length >= 2 ? '🧠' : '🤖'"></span>
-        <span class="pill-label" x-text="state.models.length >= 2 ? 'MoE' : 'Model'"></span>
-        <span class="pill-value" x-text="state.models.length + (state.models.length >= 2 ? ' active' : '')"></span>
+        <span class="pill-icon" x-text="(state.provider === 'openrouter' && state.models.length >= 2) ? '🧠' : '🤖'"></span>
+        <span class="pill-label" x-text="(state.provider === 'openrouter' && state.models.length >= 2) ? 'MoE' : 'Model'"></span>
+        <span class="pill-value" x-text="state.models.length + ((state.provider === 'openrouter' && state.models.length >= 2) ? ' active' : '')"></span>
       </button>
 
       <!-- API key pill -->
       <button @click="modal='key'" class="pill" :class="hasKey ? 'pill-ok' : 'pill-warn'"
-              :title="localKey ? 'Key stored in this tab only (sessionStorage). Click to manage.' : (state.has_env_key ? 'Using server-side OPENROUTER_API_KEY env.' : 'No key yet.')">
+              :title="localKey ? state.provider + ' key stored in this tab only. Click to manage.' : (state.provider === 'openrouter' && state.has_env_key ? 'Using server-side OPENROUTER_API_KEY env.' : 'No key yet for ' + state.provider)">
         <span class="pill-icon" x-text="hasKey ? '🔑' : '⚠'"></span>
-        <span class="pill-label">OpenRouter</span>
-        <span class="pill-value" x-text="localKey ? 'tab-only' : (state.has_env_key ? 'shared env' : 'add key')"></span>
+        <span class="pill-label capitalize" x-text="state.provider"></span>
+        <span class="pill-value" x-text="localKey ? 'tab-only' : ((state.provider === 'openrouter' && state.has_env_key) ? 'shared env' : 'add key')"></span>
       </button>
 
       <div class="flex-1"></div>
@@ -179,8 +179,8 @@
         </div>
       </section>
 
-      <!-- MoE explainer — when ≥2 models are selected -->
-      <section x-show="state.models.length >= 2 && !moeBannerDismissed && state.sentences.length > 0"
+      <!-- MoE explainer — only meaningful on OpenRouter -->
+      <section x-show="state.provider === 'openrouter' && state.models.length >= 2 && !moeBannerDismissed && state.sentences.length > 0"
                class="rounded-xl border border-amber-200 bg-amber-50/60 px-4 py-2.5 text-xs text-amber-900 flex items-center gap-3" x-transition>
         <span class="text-base">🧠</span>
         <div class="flex-1">
@@ -189,7 +189,7 @@
         </div>
         <button @click="moeBannerDismissed=true" class="btn btn-ghost btn-icon">✕</button>
       </section>
-      <section x-show="state.models.length === 1 && state.sentences.length > 0 && !moeHintDismissed"
+      <section x-show="state.provider === 'openrouter' && state.models.length === 1 && state.sentences.length > 0 && !moeHintDismissed"
                class="rounded-xl border border-ink-200 bg-white px-4 py-2.5 text-xs text-ink-700 flex items-center gap-3" x-transition>
         <span class="text-base">💡</span>
         <div class="flex-1">
@@ -247,6 +247,19 @@
             </button>
           </header>
 
+          <!-- Per-model accuracy strip (vs current consensus / your corrections) -->
+          <div class="px-4 pt-2 -mb-1 flex flex-wrap items-center gap-1.5"
+               x-show="sent.status === 'done' && Object.keys(sent.per_model || {}).length > 0">
+            <span class="text-[10px] uppercase tracking-wider text-ink-500 font-semibold mr-1" title="Per-model agreement with the current consensus (updates as you correct)">vs consensus</span>
+            <template x-for="m in modelAccuracy(sent)" :key="m.model">
+              <span class="accuracy-pill" :class="accuracyClass(m.pct)"
+                    :title="`${m.model}: ${m.correct}/${m.total} task-meaningful fields match`">
+                <span class="font-mono opacity-80" x-text="modelShort(m.model)"></span>
+                <strong x-text="m.pct + '%'"></strong>
+              </span>
+            </template>
+          </div>
+
           <div class="p-4">
             <div class="tokens-flow">
               <template x-for="(tok, tidx) in sent.tokens" :key="tidx + '-' + rev">
@@ -441,25 +454,52 @@
         <button @click="closeModal()" class="btn btn-ghost btn-icon">✕</button>
       </header>
       <div class="p-5 space-y-4">
-        <p class="text-xs text-ink-500">Pick one model for single inference, or 2+ to enable <strong>Mixture-of-Experts</strong> (parallel calls + majority vote, with disagreements highlighted on each token).</p>
-        <div class="space-y-1.5 max-h-72 overflow-y-auto">
-          <template x-for="m in allDisplayableModels()" :key="m">
-            <label class="flex items-center gap-3 p-2 rounded hover:bg-ink-50 cursor-pointer">
-              <input type="checkbox" :checked="state.models.includes(m)" @change="toggleModel(m)">
-              <span class="font-mono text-sm" x-text="m"></span>
-              <span class="badge badge-uploaded text-[10px]" x-show="!state.curated_models.includes(m)">custom</span>
-            </label>
-          </template>
+        <!-- Provider selector -->
+        <div>
+          <label class="lbl">Provider</label>
+          <div class="grid grid-cols-3 gap-2">
+            <template x-for="p in state.providers" :key="p">
+              <button class="preset-card text-xs capitalize"
+                      :class="state.provider === p ? 'preset-card-active' : ''"
+                      @click="setProvider(p)">
+                <div class="font-semibold" x-text="p"></div>
+                <div class="text-[10px] text-ink-500 mt-0.5 font-mono normal-case truncate"
+                     x-text="p === 'openrouter' ? 'openrouter.ai/api/v1 · MoE' : (p === 'mistral' ? 'api.mistral.ai/v1' : 'api.openai.com/v1')"></div>
+              </button>
+            </template>
+          </div>
+          <p class="text-[11px] text-ink-500 mt-1.5">
+            <span x-show="state.provider === 'openrouter'">🧠 OpenRouter supports MoE — you can pick several models.</span>
+            <span x-show="state.provider !== 'openrouter'">Single-model only. For Mixture-of-Experts (parallel models + vote), use <strong>OpenRouter</strong>.</span>
+          </p>
         </div>
+
+        <!-- Curated models for the active provider -->
         <div>
-          <label class="lbl">Custom OpenRouter slug</label>
+          <label class="lbl">Models for <span x-text="state.provider"></span></label>
+          <div class="space-y-1.5 max-h-60 overflow-y-auto border border-ink-200 rounded-lg p-1">
+            <template x-for="m in allDisplayableModels()" :key="m">
+              <label class="flex items-center gap-3 p-2 rounded hover:bg-ink-50 cursor-pointer">
+                <input type="checkbox" :checked="state.models.includes(m)" @change="toggleModel(m)">
+                <span class="font-mono text-sm" x-text="m"></span>
+                <span class="badge badge-uploaded text-[10px]" x-show="!state.curated_models.includes(m)">custom</span>
+              </label>
+            </template>
+          </div>
+        </div>
+
+        <!-- Custom slug -->
+        <div>
+          <label class="lbl">Add a custom model slug for <span x-text="state.provider"></span></label>
           <div class="flex gap-2 mt-1">
-            <input class="input flex-1" type="text" x-model="modelEditor.custom" placeholder="provider/model-id">
+            <input class="input flex-1" type="text" x-model="modelEditor.custom" placeholder="model-id (e.g. mistral-medium-2505)">
             <button class="btn btn-secondary" @click="addCustomModel()">Add</button>
           </div>
         </div>
-        <div>
-          <label class="lbl">MoE priority (tie-break order)</label>
+
+        <!-- MoE priority — only meaningful when OpenRouter + ≥2 models -->
+        <div x-show="state.provider === 'openrouter'">
+          <label class="lbl">MoE priority (tie-break order, comma-separated slugs)</label>
           <input class="input" type="text" x-model="modelEditor.priority"
                  @change="saveSettings({priority: modelEditor.priority.split(',').map(s => s.trim()).filter(Boolean)})"
                  placeholder="model1, model2, model3">
@@ -472,24 +512,36 @@
   <div class="modal-backdrop" x-show="modal === 'key'" @click.self="closeModal()" x-transition>
     <div class="modal-card max-w-md">
       <header class="modal-header">
-        <h2 class="font-semibold">OpenRouter API key</h2>
+        <h2 class="font-semibold capitalize"><span x-text="state.provider"></span> API key</h2>
         <button @click="closeModal()" class="btn btn-ghost btn-icon">✕</button>
       </header>
       <div class="p-5 space-y-3">
+        <!-- Provider switcher (compact) -->
+        <div>
+          <label class="lbl">Provider</label>
+          <div class="flex gap-1.5">
+            <template x-for="p in state.providers" :key="p">
+              <button class="chip capitalize"
+                      :class="state.provider === p ? 'chip-active' : ''"
+                      @click="setProvider(p)" x-text="p"></button>
+            </template>
+          </div>
+        </div>
+
         <div class="rounded-lg bg-emerald-50 border border-emerald-200 p-3 text-xs text-emerald-900">
-          <strong>🔒 Tab-scoped storage.</strong> Your key lives only in this browser tab's <code>sessionStorage</code>. It is sent as an <code>X-OpenRouter-Key</code> header on each annotation request and <strong>never stored on the server</strong>. Closing the tab wipes it. Use <strong>Clear key</strong> below at any time.
+          <strong>🔒 Tab-scoped storage.</strong> Keys live only in this browser tab's <code>sessionStorage</code> (one slot per provider). Sent as <code>X-API-Key</code> + <code>X-LLM-Provider</code> headers — never stored on the server.
         </div>
 
         <template x-if="localKey">
           <div class="rounded-lg bg-ink-50 border border-ink-200 p-3 text-xs flex items-center gap-2">
             <span class="text-emerald-700">✓</span>
-            <span class="text-ink-700 flex-1">A key is set in this tab.</span>
-            <button class="btn btn-ghost text-xs text-red-600" @click="clearKey()">Clear key</button>
+            <span class="text-ink-700 flex-1">A <span x-text="state.provider"></span> key is set in this tab.</span>
+            <button class="btn btn-ghost text-xs text-red-600" @click="clearKey()">Clear</button>
           </div>
         </template>
 
         <label class="lbl" x-text="localKey ? 'Replace with a new key' : 'Paste your key, then press Enter (or click Save)'"></label>
-        <input class="input" type="password" placeholder="sk-or-v1-…"
+        <input class="input" type="password" :placeholder="state.provider === 'openrouter' ? 'sk-or-v1-…' : (state.provider === 'openai' ? 'sk-…' : 'Mistral key')"
                x-model="keyEditor.value"
                @keydown.enter.prevent="saveKey()"
                x-ref="keyInput"
@@ -500,20 +552,25 @@
         <div class="flex items-center gap-2">
           <button class="btn btn-primary flex-1" @click="saveKey()" x-text="localKey ? 'Replace key' : 'Save key (this tab only)'"></button>
           <button class="btn btn-secondary" @click="testKey(true)" :disabled="keyEditor.testing"
-                  title="Test against OpenRouter and save if it works">
+                  :title="`Test against ${state.provider} and save if it works`">
             <span x-show="!keyEditor.testing">Test &amp; save</span>
             <span x-show="keyEditor.testing" class="spinner-sm"></span>
           </button>
         </div>
         <div class="text-xs" x-show="keyEditor.result" :class="keyEditor.ok ? 'text-emerald-700' : 'text-red-600'" x-text="keyEditor.result"></div>
 
-        <template x-if="state.has_env_key">
+        <template x-if="state.has_env_key && state.provider === 'openrouter'">
           <div class="text-[11px] text-ink-500 border-t border-ink-100 pt-2">
             ⓘ A server-side <code>OPENROUTER_API_KEY</code> is also configured. If you don't set a tab-key, the server fallback is used.
           </div>
         </template>
 
-        <p class="text-[11px] text-ink-400">Get a key → <a class="text-accent-600 underline" target="_blank" href="https://openrouter.ai/keys">openrouter.ai/keys</a></p>
+        <p class="text-[11px] text-ink-400">
+          Get a key →
+          <a class="text-accent-600 underline" target="_blank" x-show="state.provider === 'openrouter'" href="https://openrouter.ai/keys">openrouter.ai/keys</a>
+          <a class="text-accent-600 underline" target="_blank" x-show="state.provider === 'mistral'" href="https://console.mistral.ai/api-keys">console.mistral.ai/api-keys</a>
+          <a class="text-accent-600 underline" target="_blank" x-show="state.provider === 'openai'" href="https://platform.openai.com/api-keys">platform.openai.com/api-keys</a>
+        </p>
       </div>
     </div>
   </div>
@@ -831,6 +888,6 @@
     </template>
   </div>
 
-  <script src="/static/app.js?v=20260516b"></script>
+  <script src="/static/app.js?v=20260516c"></script>
 </body>
 </html>

static/styles.css

@@ -292,6 +292,20 @@ textarea.input { font-family: 'JetBrains Mono', monospace; }
 .badge-corrected { background: #d1fae5; color: #065f46; }
 .badge-uploaded { background: #fef3c7; color: #92400e; }
 
+/* ============================================================
+   Per-model accuracy pills (under each sentence)
+   ============================================================ */
+.accuracy-pill {
+  display: inline-flex; align-items: center; gap: 0.35em;
+  padding: 0.1em 0.55em;
+  border-radius: 999px;
+  font-size: 11px;
+  line-height: 1.4;
+}
+.accuracy-pill-high { background: #d1fae5; color: #065f46; border: 1px solid #6ee7b7; }
+.accuracy-pill-mid  { background: #fef3c7; color: #92400e; border: 1px solid #fcd34d; }
+.accuracy-pill-low  { background: #fee2e2; color: #b91c1c; border: 1px solid #fca5a5; }
+
 /* ============================================================
    Tables (in modals)
    ============================================================ */