Update main.py

main.py

@@ -234,62 +234,84 @@ def q_stats() -> dict:
     }
 
 # ---------------------------------------------------------------------------
-# Reward scoring
+# Reward scoring  — 0–10 float scale
 # ---------------------------------------------------------------------------
+# Scale semantics:
+#   0–1  catastrophic (PII leak, injection, critical safety failure)
+#   2–3  failure      (error, hallucinated tool, unrecoverable)
+#   4–5  partial      (slow, compensated saga, incomplete)
+#   6    acceptable   (baseline — completed without issues)
+#   7    good         (fast, used skill, memory stored)
+#   8    excellent    (all bonuses, fast, clean)
+#   9    exceptional  (auto ceiling — reserved for near-perfect)
+#   10   human-only   (PATCH /api/traces/{id}/rate override only)
+#
+# Auto-score is capped at 9.0.
+# Human rating via PATCH /api/rlhf/{id} can set 10.
+# RLHF auto-collection: score>=8 → preferred, score<=3 → rejected
+
+SCORE_BASELINE     = 6.0
+SCORE_AUTO_CEILING = 9.0
+SCORE_HUMAN_MAX    = 10.0
+
 def score_trace_event(ev: dict) -> tuple[float, dict]:
     """
-    Score a trace event → reward in [-1.0, 1.0].
+    Score a trace event on a 0–10 float scale.
     Returns (score, components).
     """
-    components = {}
-    score = 0.0
+    components: dict = {}
+    score = SCORE_BASELINE
 
-    # Base: error is always bad
+    # ── Deductions ────────────────────────────────────────────────
     if ev.get("status") == "error":
-        components["error_penalty"] = -0.4
-        score -= 0.4
+        components["error"] = -3.0
+        score -= 3.0
+
+    if ev.get("injection_detected"):
+        components["injection_detected"] = -4.0
+        score -= 4.0
+
+    if ev.get("pii_leaked"):
+        components["pii_leaked"] = -4.0
+        score -= 4.0
+
+    if ev.get("hallucinated_tool"):
+        components["hallucinated_tool"] = -3.0
+        score -= 3.0
+
+    if ev.get("saga_compensated"):
+        components["saga_compensated"] = -1.0
+        score -= 1.0
 
-    # Latency score for LLM calls (lower = better)
     lat = ev.get("latency_ms")
-    if lat is not None and ev.get("event_type") == "llm_call":
-        if lat < 500:
-            v = 0.3; components["latency_fast"] = v
-        elif lat < 1500:
-            v = 0.1; components["latency_ok"] = v
-        elif lat < 4000:
-            v = -0.1; components["latency_slow"] = v
-        else:
-            v = -0.3; components["latency_very_slow"] = v
-        score += v
-
-    # Token efficiency for LLM calls
-    tin  = ev.get("tokens_in")  or 0
-    tout = ev.get("tokens_out") or 0
-    if tin > 0 and tout > 0 and ev.get("event_type") == "llm_call":
-        ratio = tout / max(tin, 1)
-        if ratio > 0.5:
-            v = 0.1; components["token_efficiency"] = v; score += v
-        elif ratio < 0.05:
-            v = -0.05; components["token_low_output"] = v; score += v
-
-    # ReAct step: reward progress
-    if ev.get("event_type") == "react_step":
-        components["react_progress"] = 0.1
-        score += 0.1
-
-    # Skill load: reward reuse over re-implementation
+    if lat is not None and lat > 8000:
+        components["latency_over_8s"] = -1.5
+        score -= 1.5
+
+    # ── Bonuses ───────────────────────────────────────────────────
     if ev.get("event_type") == "skill_load":
-        components["skill_reuse"] = 0.15
-        score += 0.15
+        components["skill_load"] = +0.5
+        score += 0.5
 
-    # Self-reflect: always reward
-    if ev.get("event_type") == "self_reflect":
-        components["reflection_bonus"] = 0.2
-        score += 0.2
+    if ev.get("skill_candidate"):
+        components["skill_candidate"] = +1.0
+        score += 1.0
 
-    # Clamp to [-1, 1]
-    score = max(-1.0, min(1.0, score))
-    return round(score, 4), components
+    if ev.get("memory_stored"):
+        components["memory_stored"] = +0.3
+        score += 0.3
+
+    if lat is not None and lat < 1000 and ev.get("event_type") == "llm_call":
+        components["latency_under_1s"] = +0.5
+        score += 0.5
+
+    if ev.get("saga_clean"):
+        components["saga_clean"] = +0.5
+        score += 0.5
+
+    # Clamp 0–AUTO_CEILING (10 is human-only)
+    score = max(0.0, min(SCORE_AUTO_CEILING, score))
+    return round(score, 2), components
 
 # ---------------------------------------------------------------------------
 # Trace sync pipeline
@@ -387,6 +409,21 @@ def pull_and_score_traces() -> dict:
         # Q-table update: map event → (state, action)
         _update_qtable_from_trace(ev, reward)
 
+        # RLHF auto-collection: preferred (>=8) and rejected (<=3)
+        if reward >= 8.0 or reward <= 3.0:
+            label  = "approved" if reward >= 8.0 else "rejected"
+            prompt = (f"[{ev.get('agent','?')}] {ev.get('event_type','?')}: "
+                      f"{ev.get('tool_name') or ev.get('model') or ev.get('task','')}")
+            completion = json.dumps({k: ev.get(k) for k in
+                ("status","latency_ms","tokens_out","saga_clean","skill_candidate","memory_stored")
+                if ev.get(k) is not None})
+            try:
+                rlhf_add(ev.get("agent","unknown"), prompt, completion,
+                         label=label, reward=reward, source="auto",
+                         meta={"trace_id": ev["id"], "components": components})
+            except Exception:
+                pass
+
         scored += 1
         reward_sum += reward
         new_cursors[agent] = max(new_cursors.get(agent, 0), ts)
@@ -663,6 +700,10 @@ MCP_TOOLS = [
                     "properties":{"description":{"type":"string"},"agent":{"type":"string"}}}},
     {"name":"learn_sync","description":"Trigger immediate trace pull and reward scoring.",
      "inputSchema":{"type":"object","properties":{}}},
+    {"name":"learn_rate_trace","description":"Human rating override for a trace (0–10 float). Score 10 is human-only ceiling. Scores >=8 auto-labeled preferred, <=3 auto-labeled rejected in RLHF store.",
+     "inputSchema":{"type":"object","required":["trace_id","rating"],
+                    "properties":{"trace_id":{"type":"string"},"rating":{"type":"number","minimum":0,"maximum":10},
+                                  "agent":{"type":"string"},"comment":{"type":"string"}}}},
 ]
 
 def handle_mcp(method, params, req_id):
@@ -691,6 +732,24 @@ def handle_mcp(method, params, req_id):
             cid = candidate_add(a["description"],a["agent"])
             return txt({"ok":True,"id":cid})
         if n=="learn_sync":     return txt(pull_and_score_traces())
+        if n=="learn_rate_trace":
+            rating = float(a["rating"])
+            if not (0.0 <= rating <= SCORE_HUMAN_MAX):
+                return txt({"ok":False,"error":f"rating must be 0–{SCORE_HUMAN_MAX}"})
+            agent   = str(a.get("agent","unknown"))
+            comment = str(a.get("comment",""))
+            try: _http_patch(f"{TRACE_URL}/api/trace/{a['trace_id']}/reward",
+                             {"reward":rating,"source":"human","comment":comment})
+            except Exception: pass
+            label = "approved" if rating>=8.0 else ("rejected" if rating<=3.0 else "unlabeled")
+            conn = get_db()
+            conn.execute("INSERT OR IGNORE INTO rewards (id,trace_id,agent,event_type,raw_score,components,ts) VALUES (?,?,?,?,?,?,?)",
+                (str(uuid.uuid4()),a["trace_id"],agent,"human_rating",rating,
+                 json.dumps({"human_override":True,"comment":comment}),time.time()))
+            conn.commit(); conn.close()
+            rid = rlhf_add(agent,f"[human-rated] {a['trace_id']}",comment or "human override",
+                           label=label,reward=rating,source="human",meta={"trace_id":a["trace_id"]})
+            return txt({"ok":True,"trace_id":a["trace_id"],"rating":rating,"label":label,"rlhf_id":rid})
         return {"jsonrpc":"2.0","id":req_id,"error":{"code":-32601,"message":f"Unknown tool: {n}"}}
     if method in ("notifications/initialized","notifications/cancelled"): return None
     return {"jsonrpc":"2.0","id":req_id,"error":{"code":-32601,"message":f"Method not found: {method}"}}
@@ -768,6 +827,42 @@ async def api_rlhf_label(entry_id:str, request:Request):
     ok = rlhf_label(entry_id, b.get("label","unlabeled"), b.get("reward"))
     return JSONResponse({"ok":ok})
 
+@app.patch("/api/traces/{trace_id}/rate")
+async def api_trace_rate(trace_id:str, request:Request):
+    """Human rating override — allows score of 10 (human-only ceiling).
+    Writes back to agent-trace and updates Q-table."""
+    if not _auth(request): raise HTTPException(403,"Invalid X-Learn-Key")
+    b = await request.json()
+    rating = float(b.get("rating", b.get("reward", 0.0)))
+    if not (0.0 <= rating <= SCORE_HUMAN_MAX):
+        raise HTTPException(400, f"rating must be 0–{SCORE_HUMAN_MAX}")
+    agent   = str(b.get("agent","unknown"))
+    comment = str(b.get("comment",""))
+
+    # Write reward back to agent-trace (best-effort)
+    try:
+        _http_patch(f"{TRACE_URL}/api/trace/{trace_id}/reward",
+                    {"reward": rating, "source": "human", "comment": comment})
+    except Exception:
+        pass
+
+    # Log in rewards table
+    conn = get_db()
+    conn.execute("""
+        INSERT OR IGNORE INTO rewards (id,trace_id,agent,event_type,raw_score,components,ts)
+        VALUES (?,?,?,?,?,?,?)
+    """, (str(uuid.uuid4()), trace_id, agent, "human_rating",
+          rating, json.dumps({"human_override": True, "comment": comment}), time.time()))
+    conn.commit(); conn.close()
+
+    # RLHF: store as approved/rejected based on rating
+    label = "approved" if rating >= 8.0 else ("rejected" if rating <= 3.0 else "unlabeled")
+    rlhf_add(agent, f"[human-rated trace] {trace_id}", comment or "human override",
+             label=label, reward=rating, source="human",
+             meta={"trace_id": trace_id, "comment": comment})
+
+    return JSONResponse({"ok": True, "trace_id": trace_id, "rating": rating, "label": label})
+
 # --- Skill candidates ---
 @app.get("/api/candidates")
 async def api_candidates(status:str=Query("pending")):
@@ -1024,7 +1119,7 @@ function renderRewards(){
     </div>
     <div class="section">Scoring model</div>
     <div style="background:var(--sf);border:1px solid var(--br);border-radius:8px;overflow:hidden">
-      ${[['error_penalty','-0.40','Any event with status=error'],['latency_fast (LLM <500ms)','+0.30','LLM call completed quickly'],['latency_ok (500-1500ms)','+0.10','LLM call acceptable latency'],['latency_slow (1500-4000ms)','-0.10','LLM call slow'],['latency_very_slow (>4000ms)','-0.30','LLM call very slow'],['token_efficiency','+0.10','Output/input ratio > 0.5'],['react_progress','+0.10','Each ReAct step completed'],['skill_reuse','+0.15','Skill loaded from FORGE'],['reflection_bonus','+0.20','Agent performed self-reflection']].map(([k,v,d])=>`<div class="config-row"><span class="config-key">${k}</span><span class="config-val">${v}</span><span class="config-desc">${d}</span></div>`).join('')}
+      ${[['baseline','+6.0','Every event starts here (acceptable)'],['error','-3.0','status=error'],['injection_detected','-4.0','Injection flag from agent-harness'],['pii_leaked','-4.0','PII exfiltration detected by compliance'],['hallucinated_tool','-3.0','Agent called non-existent tool'],['saga_compensated','-1.0','Saga pattern ran compensations'],['latency > 8s','-1.5','LLM call took > 8000ms'],['skill_load','+0.5','Reused skill from FORGE'],['skill_candidate','+1.0','Agent surfaced a new skill pattern'],['memory_stored','+0.3','Agent stored to agent-memory'],['latency < 1s (LLM)','+0.5','LLM call completed in < 1000ms'],['saga_clean','+0.5','Saga completed without compensation'],['AUTO CEILING','9.0','Max auto-score (10 = human-only via PATCH /api/traces/{id}/rate)']].map(([k,v,d])=>`<div class="config-row"><span class="config-key">${k}</span><span class="config-val" style="color:${v.startsWith('-')?'var(--rd)':v==='9.0'?'var(--ye)':'var(--gr)'}">${v}</span><span class="config-desc">${d}</span></div>`).join('')}
     </div>`;
 }
 
@@ -1135,4 +1230,4 @@ loadAll();setInterval(loadAll,15000);
 async def root(): return HTMLResponse(content=SPA, media_type="text/html; charset=utf-8")
 
 if __name__ == "__main__":
-    uvicorn.run(app, host="0.0.0.0", port=PORT, log_level="info")
+    uvicorn.run(app, host="0.0.0.0", port=PORT, log_level="info")