{
  "config": {
    "model": "meta-llama/Llama-2-7b-chat-hf",
    "device": "cuda",
    "model_dtype": "fp16",
    "tasks": "gsm8k,commonsenseqa,strategyqa,arc_challenge,openbookqa,qasc,logiqa",
    "heldout_tasks": "gsm8k,logiqa",
    "layer": 10,
    "n_subspace": 64,
    "n_eval": 32,
    "use_forced_choice": true,
    "do_sample": false,
    "offline": true
  },
  "commands": {
    "gsm8k": [
      "/home/zs89/miniconda3/envs/flashsvd/bin/python",
      "/home/zs89/decodeshare/reasoning/disturb_CoT_shared_loto_reasoning.py",
      "--model",
      "meta-llama/Llama-2-7b-chat-hf",
      "--device",
      "cuda",
      "--model_dtype",
      "fp16",
      "--layer",
      "10",
      "--tasks",
      "gsm8k,commonsenseqa,strategyqa,arc_challenge,openbookqa,qasc,logiqa",
      "--mode",
      "loto",
      "--loto_eval_mode",
      "heldout",
      "--loto_only",
      "gsm8k",
      "--n_subspace",
      "64",
      "--n_eval",
      "32",
      "--pca_var",
      "0.95",
      "--tau",
      "0.001",
      "--m_shared",
      "all",
      "--calib_decode_max_new_tokens",
      "48",
      "--per_task_max_states",
      "2048",
      "--alpha_remove",
      "1.0",
      "--reasoning_tokens",
      "64",
      "--max_new_tokens",
      "96",
      "--temperature",
      "0.7",
      "--top_p",
      "0.9",
      "--top_k",
      "0",
      "--do_sample",
      "0",
      "--template_randomization",
      "1",
      "--template_seed",
      "1234",
      "--shuffle_choices",
      "1",
      "--add_answer_prefix",
      "1",
      "--answer_prefix",
      "\nFinal answer:",
      "--use_forced_choice",
      "1",
      "--fc_warmup_tokens",
      "0",
      "--fc_prefix_mode",
      "auto",
      "--fc_answer_prefix",
      "\nFinal answer:",
      "--batch_size",
      "2",
      "--max_prompt_len",
      "512",
      "--bootstrap_iters",
      "500",
      "--perm_iters",
      "1000",
      "--ci_alpha",
      "0.05",
      "--seed",
      "42",
      "--sample_seed",
      "12345",
      "--out_json",
      "/home/zs89/decodeshare/rebuttal/reasoning/raw/gsm8k_loto.json",
      "--out_md",
      "/home/zs89/decodeshare/rebuttal/reasoning/raw/gsm8k_loto.md"
    ],
    "logiqa": [
      "/home/zs89/miniconda3/envs/flashsvd/bin/python",
      "/home/zs89/decodeshare/reasoning/disturb_CoT_shared_loto_reasoning.py",
      "--model",
      "meta-llama/Llama-2-7b-chat-hf",
      "--device",
      "cuda",
      "--model_dtype",
      "fp16",
      "--layer",
      "10",
      "--tasks",
      "gsm8k,commonsenseqa,strategyqa,arc_challenge,openbookqa,qasc,logiqa",
      "--mode",
      "loto",
      "--loto_eval_mode",
      "heldout",
      "--loto_only",
      "logiqa",
      "--n_subspace",
      "64",
      "--n_eval",
      "32",
      "--pca_var",
      "0.95",
      "--tau",
      "0.001",
      "--m_shared",
      "all",
      "--calib_decode_max_new_tokens",
      "48",
      "--per_task_max_states",
      "2048",
      "--alpha_remove",
      "1.0",
      "--reasoning_tokens",
      "64",
      "--max_new_tokens",
      "96",
      "--temperature",
      "0.7",
      "--top_p",
      "0.9",
      "--top_k",
      "0",
      "--do_sample",
      "0",
      "--template_randomization",
      "1",
      "--template_seed",
      "1234",
      "--shuffle_choices",
      "1",
      "--add_answer_prefix",
      "1",
      "--answer_prefix",
      "\nFinal answer:",
      "--use_forced_choice",
      "1",
      "--fc_warmup_tokens",
      "0",
      "--fc_prefix_mode",
      "auto",
      "--fc_answer_prefix",
      "\nFinal answer:",
      "--batch_size",
      "2",
      "--max_prompt_len",
      "512",
      "--bootstrap_iters",
      "500",
      "--perm_iters",
      "1000",
      "--ci_alpha",
      "0.05",
      "--seed",
      "42",
      "--sample_seed",
      "12345",
      "--out_json",
      "/home/zs89/decodeshare/rebuttal/reasoning/raw/logiqa_loto.json",
      "--out_md",
      "/home/zs89/decodeshare/rebuttal/reasoning/raw/logiqa_loto.md"
    ]
  },
  "raw_files": {
    "gsm8k": {
      "json": "/home/zs89/decodeshare/rebuttal/reasoning/raw/gsm8k_loto.json",
      "md": "/home/zs89/decodeshare/rebuttal/reasoning/raw/gsm8k_loto.md"
    },
    "logiqa": {
      "json": "/home/zs89/decodeshare/rebuttal/reasoning/raw/logiqa_loto.json",
      "md": "/home/zs89/decodeshare/rebuttal/reasoning/raw/logiqa_loto.md"
    }
  },
  "per_task": [
    {
      "task": "gsm8k",
      "task_label": "Open-ended numeric reasoning",
      "n_eval": 32,
      "protocol": "generation",
      "k_eval": 77,
      "chance_acc": null,
      "baseline_near_floor": true,
      "baseline_acc": 0.0,
      "decode_shared_acc": 0.0,
      "prefill_shared_acc": 0.0,
      "random_acc": 0.0,
      "decode_minus_prefill": 0.0,
      "decode_minus_prefill_ci_low": 0.0,
      "decode_minus_prefill_ci_high": 0.0,
      "decode_minus_prefill_p": 1.0,
      "decode_minus_baseline": 0.0,
      "prefill_minus_baseline": 0.0,
      "random_minus_baseline": 0.0
    },
    {
      "task": "logiqa",
      "task_label": "Logical reasoning multiple choice",
      "n_eval": 32,
      "protocol": "forced_choice",
      "k_eval": 70,
      "chance_acc": 0.25,
      "baseline_near_floor": false,
      "baseline_acc": 0.3125,
      "decode_shared_acc": 0.15625,
      "prefill_shared_acc": 0.34375,
      "random_acc": 0.34375,
      "decode_minus_prefill": -0.1875,
      "decode_minus_prefill_ci_low": -0.3125,
      "decode_minus_prefill_ci_high": -0.0625,
      "decode_minus_prefill_p": 0.03596403596403597,
      "decode_minus_baseline": -0.15625,
      "prefill_minus_baseline": 0.03125,
      "random_minus_baseline": 0.03125
    }
  ],
  "aggregate": {
    "baseline_acc_mean": 0.15625,
    "decode_shared_acc_mean": 0.078125,
    "prefill_shared_acc_mean": 0.171875,
    "random_acc_mean": 0.171875,
    "decode_minus_baseline_mean": -0.078125,
    "prefill_minus_baseline_mean": 0.015625,
    "random_minus_baseline_mean": 0.015625,
    "decode_minus_prefill_mean": -0.09375,
    "informative_tasks": [
      "logiqa"
    ],
    "inconclusive_tasks": [
      "gsm8k"
    ]
  }
}