eval_launcher_v3.py

"""Eval launcher — checks for all 3 trained models then runs full evaluation.

Usage: python eval_launcher_v3.py
"""

import sys, time, json
from huggingface_hub import HfApi, list_repo_refs

HUB = "narcolepticchicken"
REQUIRED = [
    f"{HUB}/speculative-proposer-v3-1.7b",
    f"{HUB}/speculative-verifier-v3-4b",
    f"{HUB}/speculative-proposer-v3-8b",
]
MAX_WAIT = 7200  # 2 hours


def model_has_weights(repo_id):
    """Check if model has adapter weights pushed."""
    try:
        refs = list_repo_refs(repo_id)
        api = HfApi()
        files = api.list_repo_files(repo_id)
        has_adapter = any("adapter_model" in f for f in files)
        return has_adapter
    except Exception:
        return False


def wait_for_models():
    """Poll until all 3 models have weights."""
    start = time.time()
    while time.time() - start < MAX_WAIT:
        missing = [m for m in REQUIRED if not model_has_weights(m)]
        if not missing:
            return True
        elapsed = time.time() - start
        print(f"[{elapsed:.0f}s] Waiting for: {missing}")
        time.sleep(60)
    print("TIMEOUT: models not ready after 2h")
    return False


if __name__ == "__main__":
    print("Checking models...")
    for m in REQUIRED:
        ok = model_has_weights(m)
        print(f"  {m}: {'✓' if ok else '✗ MISSING'}")

    all_ready = all(model_has_weights(m) for m in REQUIRED)

    if not all_ready:
        print("\nWaiting for training to complete...")
        if not wait_for_models():
            sys.exit(1)

    print("\nAll models ready! Launching evaluation...")

    # Run the eval
    exec(open("eval_runner_v3.py").read())