File size: 2,204 Bytes
959dfe5 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 | #!/usr/bin/env python3
"""Structured Data Position Bias — Master Runner"""
import argparse
import logging
import os
import sys
from experiments.json_retrieval import run_json_retrieval
from experiments.table_retrieval import run_table_retrieval
from experiments.log_file_retrieval import run_log_retrieval
from src.utils import save_json
logging.basicConfig(
format="%(asctime)s - %(levelname)s - %(message)s",
level=logging.INFO,
stream=sys.stdout,
)
logger = logging.getLogger(__name__)
def parse_args():
p = argparse.ArgumentParser(description="Structured Data Position Bias")
p.add_argument("--model", default="Qwen/Qwen2.5-1.5B-Instruct")
p.add_argument("--output", default="./results")
p.add_argument("--num-items", type=int, default=100)
p.add_argument("--num-examples", type=int, default=30)
return p.parse_args()
def main():
args = parse_args()
model = args.model
out_root = args.output
os.makedirs(out_root, exist_ok=True)
logger.info("\n--- Experiment 1: JSON Array Retrieval ---")
json_results = run_json_retrieval(
model, args.num_items, args.num_examples,
os.path.join(out_root, "exp1_json"),
)
logger.info("\n--- Experiment 2: Markdown Table Retrieval ---")
table_results = run_table_retrieval(
model, args.num_items, args.num_examples,
os.path.join(out_root, "exp2_table"),
)
logger.info("\n--- Experiment 3: Log File Retrieval ---")
log_results = run_log_retrieval(
model, args.num_items, args.num_examples,
os.path.join(out_root, "exp3_log"),
)
master = {
"json": json_results,
"table": table_results,
"log": log_results,
}
save_json(os.path.join(out_root, "master_summary.json"), master)
logger.info("\n--- Structured Data PBI Comparison ---")
for exp_name, res in master.items():
depths = list(res["depths"].keys())
accs = list(res["depths"].values())
if len(accs) >= 3:
mid_idx = len(accs) // 2
pbi = (accs[0] + accs[-1]) / 2 - accs[mid_idx]
logger.info(f" {exp_name:10s} PBI={pbi:+.3f}")
if __name__ == "__main__":
main()
|