Upload eval/eval_bert_partB.py

eval/eval_bert_partB.py

@@ -0,0 +1,44 @@
+
+# ── Load models ──
+print("\n[2] Loading BERT router from Hub...")
+from transformers import AutoTokenizer, AutoModelForSequenceClassification
+import torch
+
+REPO = "narcolepticchicken/agent-cost-optimizer"
+tokenizer = AutoTokenizer.from_pretrained(REPO, subfolder="router_models/bert_router")
+bert_model = AutoModelForSequenceClassification.from_pretrained(REPO, subfolder="router_models/bert_router")
+bert_model.eval()
+print(f"  BERT loaded, num_labels={bert_model.config.num_labels}")
+
+print("\n[3] Loading v11 XGBoost router...")
+from huggingface_hub import hf_hub_download
+v11_path = hf_hub_download(REPO, "router_models/router_bundle_v11.pkl")
+v11_bundle = pickle.load(open(v11_path, "rb"))
+v11_tier_clfs = {int(k):v for k,v in v11_bundle["tier_clfs"].items()}
+v11_tier_calibs = {int(k):v for k,v in v11_bundle["tier_calibrators"].items()}
+v11_feat_keys = v11_bundle["feat_keys"]
+print(f"  v11 loaded, features={len(v11_feat_keys)}")
+
+# ── Routing functions ──
+def route_bert(problem_text):
+    inputs = tokenizer(problem_text, truncation=True, max_length=512, return_tensors="pt")
+    with torch.no_grad():
+        logits = bert_model(**inputs).logits
+    pred_class = torch.argmax(logits, dim=-1).item()
+    tier = pred_class + 1
+    probs = torch.softmax(logits, dim=-1)[0]
+    confidence = float(probs[pred_class])
+    return tier, confidence
+
+def route_v11(problem_text):
+    feats = extract_features(problem_text)
+    feat_vec = np.array([float(feats.get(k, 0.0)) for k in v11_feat_keys], dtype=np.float32).reshape(1,-1)
+    tier_probs = {}
+    for t in range(1, 6):
+        p_raw = v11_tier_clfs[t].predict_proba(feat_vec)[0, 1]
+        p_cal = float(v11_tier_calibs[t].transform([p_raw])[0])
+        tier_probs[t] = p_cal
+    for t in range(1, 6):
+        if tier_probs[t] >= 0.65:
+            return t, tier_probs[t], tier_probs
+    return 5, tier_probs[5], tier_probs