| { |
| "config": { |
| "model": "meta-llama/Llama-2-7b-chat-hf", |
| "device": "cuda", |
| "model_dtype": "fp32", |
| "layers": "28", |
| "filter_regex": "", |
| "tasks_subspace": "gsm8k,commonsenseqa,strategyqa,aqua,arc_challenge,openbookqa,qasc,logiqa", |
| "n_subspace": 128, |
| "template_seed": 1234, |
| "template_randomization": 1, |
| "shuffle_choices": 1, |
| "answer_prefix": "\nFinal answer:", |
| "add_answer_prefix": 1, |
| "calib_decode_max_new_tokens": 128, |
| "decoding": "greedy", |
| "temperature": 0.7, |
| "top_p": 0.9, |
| "top_k": 0, |
| "ks": "32,64,128,256", |
| "pca_max_rows": 200000, |
| "per_task_max_states": 20000, |
| "pca_max_dim": 4096, |
| "batch_size": 4, |
| "max_prompt_len": 512, |
| "seed": 42, |
| "out_json": "rebuttal/pca_prefill_decode_mismatch_layer28.json" |
| }, |
| "layers": [ |
| 28 |
| ], |
| "tasks_used": [ |
| "gsm8k", |
| "commonsenseqa", |
| "strategyqa", |
| "aqua", |
| "arc_challenge", |
| "openbookqa", |
| "qasc", |
| "logiqa" |
| ], |
| "by_layer": { |
| "28": { |
| "n_rows_prefill": 1024, |
| "n_rows_decode": 125624, |
| "d": 4096, |
| "k_max": 256, |
| "angles_deg": { |
| "32": { |
| "mean": 80.07994079589844, |
| "p50": 83.09337615966797, |
| "p95": 89.45713806152344, |
| "min": 38.77346420288086, |
| "max": 89.99810791015625 |
| }, |
| "64": { |
| "mean": 79.11544036865234, |
| "p50": 81.45398712158203, |
| "p95": 89.08732604980469, |
| "min": 32.79534149169922, |
| "max": 89.96986389160156 |
| }, |
| "128": { |
| "mean": 77.17108154296875, |
| "p50": 79.1217041015625, |
| "p95": 88.75955200195312, |
| "min": 27.36577033996582, |
| "max": 89.98320770263672 |
| }, |
| "256": { |
| "mean": 74.06597900390625, |
| "p50": 75.53594970703125, |
| "p95": 88.58094787597656, |
| "min": 21.949682235717773, |
| "max": 89.8911361694336 |
| } |
| }, |
| "cos_singulars": { |
| "32": { |
| "mean": 0.1674889326095581, |
| "min": 3.307082806713879e-05, |
| "max": 0.7796280384063721 |
| }, |
| "64": { |
| "mean": 0.18428365886211395, |
| "min": 0.000525902840308845, |
| "max": 0.8406106233596802 |
| }, |
| "128": { |
| "mean": 0.21747536957263947, |
| "min": 0.0002930520277004689, |
| "max": 0.8880901336669922 |
| }, |
| "256": { |
| "mean": 0.26861822605133057, |
| "min": 0.0018999958410859108, |
| "max": 0.9275124669075012 |
| } |
| }, |
| "explained_var_ratio": { |
| "32": { |
| "decode_by_decode": 0.2242752910874054, |
| "decode_by_prefill": 0.040454826437110264, |
| "prefill_by_prefill": 0.7956135344728951, |
| "prefill_by_decode": 0.0723876062212326 |
| }, |
| "64": { |
| "decode_by_decode": 0.287591268318897, |
| "decode_by_prefill": 0.05677867750419193, |
| "prefill_by_prefill": 0.8528749960118728, |
| "prefill_by_decode": 0.10226643959551017 |
| }, |
| "128": { |
| "decode_by_decode": 0.3630458220461663, |
| "decode_by_prefill": 0.08018399896361926, |
| "prefill_by_prefill": 0.9034699791539794, |
| "prefill_by_decode": 0.14957190001338527 |
| }, |
| "256": { |
| "decode_by_decode": 0.44764296315041163, |
| "decode_by_prefill": 0.11975813122324773, |
| "prefill_by_prefill": 0.9455660636505742, |
| "prefill_by_decode": 0.21092156502465595 |
| } |
| } |
| } |
| } |
| } |
