Add zero-shot baseline mode: --adapter_path none skips adapter loading, --no_think suppresses Qwen3 thinking"

evaluate_v3.py

@@ -7,50 +7,50 @@ Optimized evaluation with:
   2. Layer-aware max tokens — caps generation length per layer (saves ~40% time)
   3. Incremental saves     — writes results after every sample (never lose progress)
   4. Resume support        — skips already-evaluated IDs from a previous checkpoint
+  5. Zero-shot baseline    — --adapter_path none to evaluate base model without adapter
 
 All KPI checking logic is identical to v2 (standard-aware).
 
-Speed estimate (vs v2 on full 2521 test set):
-  v2 full test:    ~50h (2521 samples × ~70s avg)
-  v3 stratified:   ~4-5h (400 samples × ~45s avg, with tighter max_tokens)
-
 Usage:
+    # Fine-tuned model evaluation
     python evaluate_v3.py --adapter_path ./output
-    python evaluate_v3.py --adapter_path ./output --per_layer 30 --output_file my_eval.json
-    python evaluate_v3.py --adapter_path ./output --resume  # pick up where you left off
+
+    # Zero-shot baseline (no adapter, base model only)
+    python evaluate_v3.py --adapter_path none --output_file eval_v3_baseline.json
+
+    # Zero-shot baseline with Qwen3 thinking disabled
+    python evaluate_v3.py --adapter_path none --no_think --output_file eval_v3_baseline.json
+
+    # Resume interrupted run
+    python evaluate_v3.py --adapter_path ./output --resume
 """
 
 import argparse, json, re, os, sys, math, time, random, torch
 from collections import defaultdict
 from datasets import load_dataset
 from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
-from peft import PeftModel
 
 
 # ═══════════════════════════════════════════════════════════════════════
 # LAYER-AWARE MAX TOKENS
 # ═══════════════════════════════════════════════════════════════════════
-# Derived from reference output lengths in the dataset.
-# Each value = ceil(max_reference_length_chars / 3.5) + 20% safety margin.
-# This prevents the model from wasting time generating 4096 tokens for a
-# CAMARA output that's always ~700 chars (~250 tokens).
 
 LAYER_MAX_TOKENS = {
-    "tmf921":                       1600,   # ref ~4200 chars → ~1200 tok + margin
-    "intent_3gpp":                  900,    # ref ~2100 chars → ~600 tok + margin
-    "camara":                       400,    # ref ~700 chars → ~200 tok + margin
-    "a1_policy":                    350,    # ref ~650 chars → ~185 tok + margin
-    "o1_nrm":                       500,    # ref ~1050 chars → ~300 tok + margin
-    "etsi_zsm":                     1100,   # ref ~2800 chars → ~800 tok + margin
-    "tmf921_lifecycle_activate":    250,    # ref ~300 chars
-    "tmf921_lifecycle_modify":      600,    # ref ~1250 chars
-    "tmf921_lifecycle_monitor":     500,    # ref ~1000 chars
-    "tmf921_lifecycle_report":      800,    # ref ~1800 chars
-    "tmf921_lifecycle_resume":      250,    # ref ~300 chars
-    "tmf921_lifecycle_scale":       350,    # ref ~660 chars
-    "tmf921_lifecycle_suspend":     250,    # ref ~300 chars
-    "tmf921_lifecycle_terminate":   250,    # ref ~320 chars
-    "adversarial_ambiguous":        200,    # ref ~200 chars
+    "tmf921":                       1600,
+    "intent_3gpp":                  900,
+    "camara":                       400,
+    "a1_policy":                    350,
+    "o1_nrm":                       500,
+    "etsi_zsm":                     1100,
+    "tmf921_lifecycle_activate":    250,
+    "tmf921_lifecycle_modify":      600,
+    "tmf921_lifecycle_monitor":     500,
+    "tmf921_lifecycle_report":      800,
+    "tmf921_lifecycle_resume":      250,
+    "tmf921_lifecycle_scale":       350,
+    "tmf921_lifecycle_suspend":     250,
+    "tmf921_lifecycle_terminate":   250,
+    "adversarial_ambiguous":        200,
     "adversarial_contradictory":    200,
     "adversarial_out_of_scope":     200,
 }
@@ -68,7 +68,8 @@ def log(msg: str):
 def parse_args():
     p = argparse.ArgumentParser(description="TMF921 Evaluation v3 — stratified, incremental, fast")
     p.add_argument("--base_model", type=str, default="Qwen/Qwen3-8B")
-    p.add_argument("--adapter_path", type=str, default="./output")
+    p.add_argument("--adapter_path", type=str, default="./output",
+                   help="Path to LoRA adapter, or 'none' for zero-shot baseline")
     p.add_argument("--dataset", type=str, default="nraptisss/TMF921-intent-to-config-augmented")
     p.add_argument("--split", type=str, default="test")
     p.add_argument("--per_layer", type=int, default=50,
@@ -79,6 +80,8 @@ def parse_args():
                    help="Resume from existing output_file, skipping already-evaluated IDs")
     p.add_argument("--flash_attn", action="store_true", default=True)
     p.add_argument("--no_flash_attn", action="store_true", default=False)
+    p.add_argument("--no_think", action="store_true", default=False,
+                   help="Suppress Qwen3 thinking mode by adding /no_think to the last user message")
     p.add_argument("--save_generations", action="store_true", default=True)
     return p.parse_args()
 
@@ -88,18 +91,10 @@ def parse_args():
 # ═══════════════════════════════════════════════════════════════════════
 
 def stratified_sample(ds, per_layer: int, seed: int = 42):
-    """
-    Sample up to `per_layer` examples per target_layer.
-    Layers with fewer samples than `per_layer` are included in full.
-    Returns list of indices into the original dataset.
-    """
     rng = random.Random(seed)
-
-    # Group indices by target_layer
     layer_indices = defaultdict(list)
     for i in range(len(ds)):
         layer_indices[ds[i]["target_layer"]].append(i)
-
     selected = []
     layer_counts = {}
     for layer, indices in sorted(layer_indices.items()):
@@ -109,19 +104,20 @@ def stratified_sample(ds, per_layer: int, seed: int = 42):
             chosen = rng.sample(indices, per_layer)
         selected.extend(chosen)
         layer_counts[layer] = len(chosen)
-
-    # Shuffle so we don't evaluate all of one layer before the next
     rng.shuffle(selected)
-
     return selected, layer_counts
 
 
 # ═══════════════════════════════════════════════════════════════════════
-# JSON PARSING (identical to v2)
+# JSON PARSING
 # ═══════════════════════════════════════════════════════════════════════
 
 def try_parse_json(text: str):
     text = text.strip()
+    # Strip thinking tags if present
+    think_match = re.match(r"<think>[\s\S]*?</think>\s*", text)
+    if think_match:
+        text = text[think_match.end():].strip()
     if text.startswith("```"):
         text = re.sub(r"^```(?:json)?\s*\n?", "", text)
         text = re.sub(r"\n?```\s*$", "", text)
@@ -139,7 +135,7 @@ def try_parse_json(text: str):
 
 
 # ═══════════════════════════════════════════════════════════════════════
-# KPI CHECKING (identical to v2)
+# KPI CHECKING (standard-aware, identical to v2)
 # ═══════════════════════════════════════════════════════════════════════
 
 def _num_representations(val: float) -> list:
@@ -150,7 +146,6 @@ def _num_representations(val: float) -> list:
     reps.append(f"{val:.0f}")
     return list(set(reps))
 
-
 def _reliability_representations(rel_pct: float) -> list:
     reps = _num_representations(rel_pct)
     per = 1 - rel_pct / 100
@@ -169,7 +164,6 @@ def _reliability_representations(rel_pct: float) -> list:
             reps.append(f"{per:.10f}".rstrip("0").rstrip("."))
     return list(set(reps))
 
-
 def _find_all_numbers(parsed) -> list:
     nums = []
     if isinstance(parsed, dict):
@@ -186,7 +180,6 @@ def _find_all_numbers(parsed) -> list:
                 nums.extend(_find_all_numbers(item))
     return nums
 
-
 def _check_kpi_direct(parsed, row, flat):
     return {
         "has_latency":       any(r in flat for r in _num_representations(row["latency_ms"])),
@@ -196,7 +189,6 @@ def _check_kpi_direct(parsed, row, flat):
         "has_max_ues":       any(r in flat for r in _num_representations(float(row["max_ues"]))),
     }
 
-
 def _check_kpi_a1_policy(parsed, row, flat):
     results = {}
     all_nums = _find_all_numbers(parsed)
@@ -216,7 +208,6 @@ def _check_kpi_a1_policy(parsed, row, flat):
     results["has_max_ues"] = '"scope"' in flat or '"groupid"' in flat
     return results
 
-
 def _check_kpi_o1_nrm(parsed, row, flat):
     return {
         "has_latency":       '"rrmpolicy"' in flat or '"nrcelldu"' in flat,
@@ -227,7 +218,6 @@ def _check_kpi_o1_nrm(parsed, row, flat):
         "has_max_ues":       '"rrmpolicymemberlist"' in flat or '"snssai"' in flat,
     }
 
-
 DIRECT_KPI_LAYERS = {"tmf921", "intent_3gpp", "camara", "etsi_zsm"}
 
 def check_kpi_fields(parsed, row, target_layer):
@@ -242,7 +232,7 @@ def check_kpi_fields(parsed, row, target_layer):
 
 
 # ═══════════════════════════════════════════════════════════════════════
-# STRUCTURE CHECKING (identical to v2)
+# STRUCTURE CHECKING
 # ═══════════════════════════════════════════════════════════════════════
 
 LAYER_ROOT_KEYS = {
@@ -290,20 +280,16 @@ def check_structure(parsed, target_layer):
 # ═══════════════════════════════════════════════════════════════════════
 
 def save_checkpoint(output_file, config, results):
-    """Write full results JSON after every sample."""
     n = len(results)
     if n == 0:
         return
-
     total_valid = sum(1 for r in results if r["json_valid"])
     total_struct = sum(1 for r in results if r["structure_correct"])
-
     overall = {
         "total_samples": n,
         "json_validity_rate": total_valid / n,
         "structure_correctness_rate": total_struct / n,
     }
-
     kpi_fields = ["has_latency", "has_reliability", "has_dl_throughput", "has_ul_throughput", "has_max_ues"]
     kpi_samples = [r for r in results if any(k in r for k in kpi_fields)]
     if kpi_samples:
@@ -312,7 +298,6 @@ def save_checkpoint(output_file, config, results):
             overall[field + "_rate"] = sum(vals) / len(vals)
         all_kpi = [all(r.get(f, False) for f in kpi_fields) for r in kpi_samples]
         overall["all_kpis_correct_rate"] = sum(all_kpi) / len(all_kpi)
-
     per_layer = defaultdict(lambda: defaultdict(list))
     for r in results:
         layer = r["target_layer"]
@@ -321,7 +306,6 @@ def save_checkpoint(output_file, config, results):
         for k in kpi_fields:
             if k in r:
                 per_layer[layer][k].append(r[k])
-
     layer_summary = {}
     for layer, metrics in per_layer.items():
         ln = len(metrics["json_valid"])
@@ -333,22 +317,18 @@ def save_checkpoint(output_file, config, results):
         for k in kpi_fields:
             if k in metrics and metrics[k]:
                 layer_summary[layer][k] = sum(metrics[k]) / len(metrics[k])
-
     output = {
         "config": config,
         "overall": overall,
         "per_layer": layer_summary,
         "raw_results": results,
     }
-
     tmp = output_file + ".tmp"
     with open(tmp, "w") as f:
         json.dump(output, f, indent=2, default=str)
     os.replace(tmp, output_file)
 
-
 def load_checkpoint(output_file):
-    """Load previously evaluated IDs for resume."""
     if not os.path.exists(output_file):
         return [], set()
     try:
@@ -380,7 +360,6 @@ def compute_gt_baseline(ds):
         kpi = check_kpi_fields(parsed, row, layer)
         for k, v in kpi.items():
             gt_results[layer][k].append(v)
-
     log("\n  Ground-truth baseline (metric ceiling):")
     log(f"  {'Layer':<20} {'latency':>8} {'reliab':>8} {'dl_tput':>8} {'ul_tput':>8} {'max_ues':>8}")
     log("  " + "─" * 55)
@@ -402,12 +381,18 @@ def main():
     if args.no_flash_attn:
         args.flash_attn = False
 
+    # Detect baseline mode
+    is_baseline = args.adapter_path.lower() in ("none", "baseline", "base", "")
+
     log("=" * 70)
     log("TMF921 Intent Translation — Evaluation v3")
+    if is_baseline:
+        log("  *** ZERO-SHOT BASELINE MODE (no adapter) ***")
     log("  Stratified sampling · Layer-aware max tokens · Incremental saves")
     log("=" * 70)
     log(f"  Base model   : {args.base_model}")
-    log(f"  Adapter      : {args.adapter_path}")
+    log(f"  Adapter      : {'NONE (zero-shot baseline)' if is_baseline else args.adapter_path}")
+    log(f"  No-think     : {args.no_think}")
     log(f"  Dataset      : {args.dataset} [{args.split}]")
     log(f"  Per-layer N  : {args.per_layer} (-1 = all)")
     log(f"  Output       : {args.output_file}")
@@ -469,18 +454,31 @@ def main():
         model_kwargs["attn_implementation"] = "flash_attention_2"
 
     base_model = AutoModelForCausalLM.from_pretrained(args.base_model, **model_kwargs)
-    model = PeftModel.from_pretrained(base_model, args.adapter_path)
+
+    if is_baseline:
+        model = base_model
+        log("  ✅ Base model loaded (zero-shot baseline — no adapter)")
+    else:
+        from peft import PeftModel
+        model = PeftModel.from_pretrained(base_model, args.adapter_path)
+        log(f"  ✅ Fine-tuned model loaded (adapter: {args.adapter_path})")
+
     model.eval()
 
     tokenizer = AutoTokenizer.from_pretrained(args.base_model, trust_remote_code=True)
     if tokenizer.pad_token is None:
         tokenizer.pad_token = tokenizer.eos_token
 
-    log("✅ Model loaded\n")
+    log("")
 
     # ── Inference loop ──
     results = list(prev_results)
-    config = {**vars(args), "total_selected": len(indices), "layer_counts": dict(layer_counts)}
+    config_dict = {
+        **vars(args),
+        "is_baseline": is_baseline,
+        "total_selected": len(indices),
+        "layer_counts": dict(layer_counts),
+    }
     t_start = time.time()
     total_to_do = len(remaining_indices)
 
@@ -494,7 +492,18 @@ def main():
         messages = row["messages"]
         reference_output = messages[-1]["content"]
 
+        # Build prompt messages (system + user, no assistant)
         prompt_messages = [m for m in messages if m["role"] != "assistant"]
+
+        # Optionally suppress Qwen3 thinking mode
+        if args.no_think and prompt_messages:
+            last_msg = prompt_messages[-1]
+            if last_msg["role"] == "user" and "/no_think" not in last_msg["content"]:
+                prompt_messages[-1] = {
+                    "role": last_msg["role"],
+                    "content": last_msg["content"] + " /no_think"
+                }
+
         input_text = tokenizer.apply_chat_template(
             prompt_messages, tokenize=False, add_generation_prompt=True
         )
@@ -537,7 +546,7 @@ def main():
         results.append(result)
 
         # ── Incremental save ──
-        save_checkpoint(args.output_file, config, results)
+        save_checkpoint(args.output_file, config_dict, results)
 
         # ── Progress ──
         elapsed = time.time() - t_start
@@ -552,7 +561,8 @@ def main():
         s = "✓" if has_correct_structure else "✗"
         progress_n = len(done_ids) + done_now
         progress_total = len(indices)
-        log(f"  [{progress_n:>4}/{progress_total}] {target_layer:<30} JSON:{j} Struct:{s} "
+        mode_tag = "[BASE]" if is_baseline else "[FT]"
+        log(f"  {mode_tag} [{progress_n:>4}/{progress_total}] {target_layer:<30} JSON:{j} Struct:{s} "
             f"| {sample_time:.1f}s (max_tok={max_tokens}) | ETA: {eta_h}h{eta_m:02d}m")
 
     # ── Final summary ──
@@ -562,7 +572,8 @@ def main():
     total_struct = sum(1 for r in results if r["structure_correct"])
 
     log(f"\n{'=' * 70}")
-    log(f"FINAL RESULTS ({n} samples, {total_time/3600:.1f}h)")
+    mode_str = "ZERO-SHOT BASELINE" if is_baseline else "FINE-TUNED"
+    log(f"FINAL RESULTS — {mode_str} ({n} samples, {total_time/3600:.1f}h)")
     log(f"{'=' * 70}")
     log(f"  JSON Validity:        {total_valid}/{n} ({total_valid/n*100:.1f}%)")
     log(f"  Structure Correct:    {total_struct}/{n} ({total_struct/n*100:.1f}%)")