Add HumanEval evaluation script

eval_humaneval.py

@@ -0,0 +1,171 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+HumanEval Evaluation Script
+============================
+Evaluate your fine-tuned Code LLM on the HumanEval benchmark.
+
+Usage:
+    python eval_humaneval.py --model YOUR_USERNAME/code-qwen2.5-coder-3b
+
+Requirements:
+    pip install transformers peft bitsandbytes accelerate human_eval
+"""
+
+import argparse
+import os
+import torch
+from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
+from peft import PeftModel
+
+
+def load_model(model_path, base_model="Qwen/Qwen2.5-Coder-3B"):
+    print(f"📥 載入模型: {model_path}")
+    print(f"   基礎模型: {base_model}")
+
+    tokenizer = AutoTokenizer.from_pretrained(base_model)
+    if tokenizer.pad_token is None:
+        tokenizer.pad_token = tokenizer.eos_token
+
+    bnb_config = BitsAndBytesConfig(
+        load_in_4bit=True, bnb_4bit_quant_type="nf4", bnb_4bit_compute_dtype=torch.bfloat16,
+    )
+
+    base = AutoModelForCausalLM.from_pretrained(
+        base_model, quantization_config=bnb_config, device_map="auto", trust_remote_code=True,
+    )
+
+    try:
+        model = PeftModel.from_pretrained(base, model_path)
+        print("✅ LoRA adapter 已載入")
+    except Exception:
+        model = base
+        print("⚠️  未找到 adapter，使用基礎模型")
+
+    model.eval()
+    return model, tokenizer
+
+
+def generate_completion(model, tokenizer, prompt, max_new_tokens=512):
+    inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
+    with torch.no_grad():
+        outputs = model.generate(
+            **inputs, max_new_tokens=max_new_tokens, do_sample=False,
+            temperature=1.0, top_p=1.0,
+            pad_token_id=tokenizer.pad_token_id, eos_token_id=tokenizer.eos_token_id,
+        )
+    generated = outputs[0][inputs["input_ids"].shape[1]:]
+    completion = tokenizer.decode(generated, skip_special_tokens=True)
+
+    lines = completion.split("\n")
+    result_lines = []
+    for line in lines:
+        if result_lines and line.strip() and not line.startswith(" ") and not line.startswith("\t"):
+            break
+        result_lines.append(line)
+    return "\n".join(result_lines)
+
+
+def run_manual_eval(model, tokenizer):
+    print("\n" + "="*60)
+    print("  MANUAL CODE GENERATION TEST")
+    print("="*60)
+
+    test_cases = [
+        {"name": "Two Sum", "prompt": 'def two_sum(nums: list[int], target: int) -> list[int]:\n    """Given an array of integers nums and an integer target, return indices of the two numbers that add up to target."""\n'},
+        {"name": "Fibonacci", "prompt": 'def fibonacci(n: int) -> int:\n    """Return the nth Fibonacci number."""\n'},
+        {"name": "Binary Search", "prompt": 'def binary_search(arr: list[int], target: int) -> int:\n    """Return the index of target in sorted array arr, or -1 if not found."""\n'},
+        {"name": "Reverse Linked List", "prompt": 'class ListNode:\n    def __init__(self, val=0, next=None):\n        self.val = val\n        self.next = next\n\ndef reverse_linked_list(head: ListNode) -> ListNode:\n    """Reverse a singly linked list and return the new head."""\n'},
+        {"name": "Merge Sort", "prompt": 'def merge_sort(arr: list[int]) -> list[int]:\n    """Sort an array using merge sort algorithm."""\n'},
+    ]
+
+    results = []
+    for i, tc in enumerate(test_cases):
+        print(f"\n{'─'*60}")
+        print(f"📝 Test {i+1}/{len(test_cases)}: {tc['name']}")
+        print(f"{'─'*60}")
+
+        completion = generate_completion(model, tokenizer, tc["prompt"])
+        full_code = tc["prompt"] + completion
+        print(f"Generated:\n{full_code}")
+
+        try:
+            compile(full_code, "<string>", "exec")
+            print("✅ 語法正確")
+            results.append(True)
+        except SyntaxError as e:
+            print(f"❌ 語法錯誤: {e}")
+            results.append(False)
+
+    passed = sum(results)
+    print(f"\n{'='*60}")
+    print(f"  結果: {passed}/{len(results)} 語法正確 ({100*passed/len(results):.0f}%)")
+    print(f"{'='*60}")
+    return passed, len(results)
+
+
+def run_humaneval(model, tokenizer):
+    try:
+        from human_eval.data import read_problems
+        from human_eval.evaluation import evaluate_functional_correctness
+        import tempfile, json
+    except ImportError:
+        print("\n⚠️  human_eval 未安裝，跳過 HumanEval 基準測試")
+        print("   安裝方式: pip install git+https://github.com/openai/human-eval.git")
+        return None
+
+    print("\n" + "="*60)
+    print("  HUMANEVAL BENCHMARK")
+    print("="*60)
+
+    problems = read_problems()
+    print(f"📊 共 {len(problems)} 道題目")
+
+    samples = []
+    for task_id, problem in problems.items():
+        completion = generate_completion(model, tokenizer, problem["prompt"], max_new_tokens=512)
+        samples.append({"task_id": task_id, "completion": completion})
+        idx = int(task_id.split("/")[-1])
+        if (idx + 1) % 20 == 0:
+            print(f"   Generated {idx + 1}/{len(problems)}...")
+
+    with tempfile.NamedTemporaryFile(mode="w", suffix=".jsonl", delete=False) as f:
+        for sample in samples:
+            f.write(json.dumps(sample) + "\n")
+        tmp_path = f.name
+
+    print("🔄 執行功能性正確性測試...")
+    results = evaluate_functional_correctness(tmp_path)
+    pass_at_1 = results.get("pass@1", 0)
+    print(f"\n🎯 HumanEval pass@1: {pass_at_1*100:.1f}%")
+    os.unlink(tmp_path)
+    return pass_at_1
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Evaluate Code LLM")
+    parser.add_argument("--model", type=str, default="./output_code", help="Model path or HF model ID")
+    parser.add_argument("--base_model", type=str, default="Qwen/Qwen2.5-Coder-3B", help="Base model name")
+    parser.add_argument("--skip_humaneval", action="store_true", help="Skip HumanEval, only run manual tests")
+    args = parser.parse_args()
+
+    print("""
+    ╔════════════════════════════════════════════════════════════╗
+    ║          Code LLM - Evaluation                            ║
+    ╚════════════════════════════════════════════════════════════╝
+    """)
+
+    model, tokenizer = load_model(args.model, args.base_model)
+    passed, total = run_manual_eval(model, tokenizer)
+
+    if not args.skip_humaneval:
+        run_humaneval(model, tokenizer)
+    else:
+        print("\n⏭️  跳過 HumanEval 基準測試")
+
+    print("\n" + "="*60)
+    print("  EVALUATION COMPLETE")
+    print("="*60)
+
+if __name__ == "__main__":
+    main()