run_all.py

#!/usr/bin/env python3
"""Structured Data Position Bias — Master Runner"""
import argparse
import logging
import os
import sys

from experiments.json_retrieval import run_json_retrieval
from experiments.table_retrieval import run_table_retrieval
from experiments.log_file_retrieval import run_log_retrieval
from src.utils import save_json

logging.basicConfig(
    format="%(asctime)s - %(levelname)s - %(message)s",
    level=logging.INFO,
    stream=sys.stdout,
)
logger = logging.getLogger(__name__)


def parse_args():
    p = argparse.ArgumentParser(description="Structured Data Position Bias")
    p.add_argument("--model", default="Qwen/Qwen2.5-1.5B-Instruct")
    p.add_argument("--output", default="./results")
    p.add_argument("--num-items", type=int, default=100)
    p.add_argument("--num-examples", type=int, default=30)
    return p.parse_args()


def main():
    args = parse_args()
    model = args.model
    out_root = args.output
    os.makedirs(out_root, exist_ok=True)

    logger.info("\n--- Experiment 1: JSON Array Retrieval ---")
    json_results = run_json_retrieval(
        model, args.num_items, args.num_examples,
        os.path.join(out_root, "exp1_json"),
    )

    logger.info("\n--- Experiment 2: Markdown Table Retrieval ---")
    table_results = run_table_retrieval(
        model, args.num_items, args.num_examples,
        os.path.join(out_root, "exp2_table"),
    )

    logger.info("\n--- Experiment 3: Log File Retrieval ---")
    log_results = run_log_retrieval(
        model, args.num_items, args.num_examples,
        os.path.join(out_root, "exp3_log"),
    )

    master = {
        "json": json_results,
        "table": table_results,
        "log": log_results,
    }
    save_json(os.path.join(out_root, "master_summary.json"), master)

    logger.info("\n--- Structured Data PBI Comparison ---")
    for exp_name, res in master.items():
        depths = list(res["depths"].keys())
        accs = list(res["depths"].values())
        if len(accs) >= 3:
            mid_idx = len(accs) // 2
            pbi = (accs[0] + accs[-1]) / 2 - accs[mid_idx]
            logger.info(f"  {exp_name:10s} PBI={pbi:+.3f}")


if __name__ == "__main__":
    main()