data/compress_validation/compression_validation.json

[
  {
    "gamma": 0.6276459084140061,
    "log_A": -2.864020841938658,
    "R2": 0.977314,
    "theta": 10000,
    "corpus": "mongo",
    "model": "google/gemma-2-9b-it",
    "attn_empirical": {
      "10": 0.017953182898410077,
      "20": 0.010547844236328577,
      "30": 0.007733995182853605,
      "50": 0.004784861977936493,
      "100": 0.0028697990643559023,
      "200": 0.0017543701516721437,
      "500": 0.0011422816235285103,
      "1000": 0.000969841086035659,
      "2000": 0.0004331948929355652
    },
    "distances_fit": [
      30,
      50,
      100,
      200,
      500,
      1000,
      2000
    ],
    "stem": "google--gemma-2-9b-it_mongo",
    "mape": 11.143181244037981,
    "d90_th": 1524,
    "d90_em": 1529,
    "err_pct": 0.3270111183780248,
    "dH_90": -0.02993155667320758,
    "ratio90": 0.762
  },
  {
    "gamma": 0.6741618914822415,
    "log_A": -3.179715570803609,
    "R2": 0.999287,
    "theta": 10000,
    "corpus": "mongo",
    "model": "EleutherAI/pythia-2.8b",
    "attn_empirical": {
      "10": 0.008272597978745277,
      "20": 0.005688730217193047,
      "30": 0.004169839709583256,
      "50": 0.003013845277988973,
      "100": 0.0018553994908062225
    },
    "distances_fit": [
      30,
      50,
      100
    ],
    "stem": "EleutherAI--pythia-2.8b_mongo",
    "mape": 0.8329977233081147,
    "d90_th": 1476,
    "d90_em": 81,
    "err_pct": 1722.2222222222222,
    "dH_90": -0.033000193771156476,
    "ratio90": 0.738
  },
  {
    "gamma": 0.7476017873166874,
    "log_A": -2.391511197086578,
    "R2": 0.984269,
    "theta": 10000,
    "corpus": "mongo",
    "model": "EleutherAI/pythia-70m",
    "attn_empirical": {
      "10": 0.015389821239643628,
      "20": 0.011693157334811985,
      "30": 0.007082567571972807,
      "50": 0.0054180885271893605,
      "100": 0.002821484198017667,
      "200": 0.0018563934957556841,
      "500": 0.0007652510016729745,
      "1000": 0.0004260873921642391,
      "2000": 0.00039341128339098454
    },
    "distances_fit": [
      30,
      50,
      100,
      200,
      500,
      1000,
      2000
    ],
    "stem": "EleutherAI--pythia-70m_mongo",
    "mape": 11.302149727500431,
    "d90_th": 1383,
    "d90_em": 1590,
    "err_pct": 13.018867924528301,
    "dH_90": -0.03897260917359724,
    "ratio90": 0.6915
  },
  {
    "gamma": 0.8266242679750889,
    "log_A": -2.9325874169558817,
    "R2": 0.993628,
    "theta": 10000,
    "corpus": "random",
    "model": "meta-llama/Llama-2-7b-hf",
    "attn_empirical": {
      "10": 0.005234783389605581,
      "20": 0.0032586656397001613,
      "30": 0.0027830680216559107,
      "50": 0.0020982515016415466,
      "100": 0.0013744194064444551,
      "200": 0.0006745984775221183,
      "500": 0.0003505304501029766,
      "1000": 0.00016617169836536053,
      "2000": 9.237132345636685e-05
    },
    "distances_fit": [
      30,
      50,
      100,
      200,
      500,
      1000,
      2000
    ],
    "stem": "meta-llama--Llama-2-7b-hf_random",
    "mape": 7.805256961731357,
    "d90_th": 1254,
    "d90_em": 1381,
    "err_pct": 9.196234612599566,
    "dH_90": -0.04731642837652837,
    "ratio90": 0.627
  },
  {
    "gamma": 0.8296009929924347,
    "log_A": -2.376214984270495,
    "R2": 0.996923,
    "theta": 10000,
    "corpus": "random",
    "model": "mistralai/Mistral-7B-v0.1",
    "attn_empirical": {
      "10": 0.009985529906633829,
      "20": 0.0060750810507064065,
      "30": 0.005252508784696045,
      "50": 0.0036803160721643104,
      "100": 0.0022635483171325175,
      "200": 0.0010285094767136292,
      "500": 0.0005741028257438707,
      "1000": 0.00029642416384174594,
      "2000": 0.0001669441466219723
    },
    "distances_fit": [
      30,
      50,
      100,
      200,
      500,
      1000,
      2000
    ],
    "stem": "mistralai--Mistral-7B-v0.1_random",
    "mape": 5.478842947828659,
    "d90_th": 1248,
    "d90_em": 1406,
    "err_pct": 11.2375533428165,
    "dH_90": -0.04770621251295322,
    "ratio90": 0.624
  },
  {
    "gamma": 0.9311078627189842,
    "log_A": -2.350543685121484,
    "R2": 0.983104,
    "theta": 10000,
    "corpus": "mongo",
    "model": "EleutherAI/pythia-1b",
    "attn_empirical": {
      "10": 0.010958388174573582,
      "20": 0.006365462491909664,
      "30": 0.003956688532812728,
      "50": 0.0027740057195640272,
      "100": 0.0014326052833348512,
      "200": 0.0006671193842258718,
      "500": 0.00020134633510477014,
      "1000": 0.00015050364037354788,
      "2000": 0.00010232122304538885
    },
    "distances_fit": [
      30,
      50,
      100,
      200,
      500,
      1000,
      2000
    ],
    "stem": "EleutherAI--pythia-1b_mongo",
    "mape": 13.09086093814886,
    "d90_th": 1028,
    "d90_em": 1370,
    "err_pct": 24.963503649635037,
    "dH_90": -0.062104239203459546,
    "ratio90": 0.514
  },
  {
    "gamma": 0.9966953735480816,
    "log_A": -2.1584093095473813,
    "R2": 0.993942,
    "theta": 1000000,
    "corpus": "mongo",
    "model": "Qwen/Qwen2.5-7B",
    "attn_empirical": {
      "10": 0.01053441942591841,
      "20": 0.005741046461561281,
      "30": 0.003944915988637755,
      "50": 0.002418298812634829,
      "100": 0.0012223406427882666,
      "200": 0.0004989972161840544,
      "500": 0.0002234310967226823,
      "1000": 0.00014713535652036727,
      "2000": 5.419184616888136e-05
    },
    "distances_fit": [
      30,
      50,
      100,
      200,
      500,
      1000,
      2000
    ],
    "stem": "Qwen--Qwen2.5-7B_mongo",
    "mape": 8.69563111859062,
    "d90_th": 856,
    "d90_em": 990,
    "err_pct": 13.535353535353536,
    "dH_90": -0.0008081142096023077,
    "ratio90": 0.428
  },
  {
    "gamma": 1.003714187534367,
    "log_A": -1.6477058895472607,
    "R2": 0.977698,
    "theta": null,
    "corpus": "random",
    "model": "EleutherAI/pythia-14m",
    "attn_empirical": {
      "10": 0.010373582980052257,
      "20": 0.006793614017466704,
      "30": 0.005060315049162858,
      "50": 0.0042915507254656406,
      "100": 0.0019918971036935952,
      "200": 0.0011028067811053512,
      "500": 0.00045950785397306186,
      "1000": 0.00012169795030211442,
      "2000": 0.00010627185478203357
    },
    "distances_fit": [
      30,
      50,
      100,
      200,
      500,
      1000,
      2000
    ],
    "stem": "EleutherAI--pythia-14m_random",
    "mape": 20.067563393050918,
    "d90_th": 836,
    "d90_em": 1225,
    "err_pct": 31.755102040816325,
    "dH_90": -0.07484146013435433,
    "ratio90": 0.418
  },
  {
    "gamma": 1.0171452847779678,
    "log_A": -1.8268598516498322,
    "R2": 0.981723,
    "theta": 10000,
    "corpus": "random",
    "model": "EleutherAI/pythia-160m",
    "attn_empirical": {
      "10": 0.010829462509912749,
      "20": 0.009873490159192847,
      "30": 0.003808915811306279,
      "50": 0.003134424632622136,
      "100": 0.001589946753811091,
      "200": 0.0009230124831406607,
      "500": 0.0003505013234017598,
      "1000": 0.00015388178632242167,
      "2000": 5.142585230411594e-05
    },
    "distances_fit": [
      30,
      50,
      100,
      200,
      500,
      1000,
      2000
    ],
    "stem": "EleutherAI--pythia-160m_random",
    "mape": 17.944445910277434,
    "d90_th": 799,
    "d90_em": 870,
    "err_pct": 8.160919540229886,
    "dH_90": -0.07731478366925829,
    "ratio90": 0.3995
  },
  {
    "gamma": 1.0218530106365162,
    "log_A": -1.7669627940483377,
    "R2": 0.981594,
    "theta": 10000,
    "corpus": "mongo",
    "model": "EleutherAI/pythia-410m",
    "attn_empirical": {
      "10": 0.010637531184022211,
      "20": 0.007524669199354119,
      "30": 0.004181321816156721,
      "50": 0.0032434275430083897,
      "100": 0.001533225679288282,
      "200": 0.000936404794475594,
      "500": 0.0003803397886238397,
      "1000": 0.0001626184749986553,
      "2000": 5.13333430424407e-05
    },
    "distances_fit": [
      30,
      50,
      100,
      200,
      500,
      1000,
      2000
    ],
    "stem": "EleutherAI--pythia-410m_mongo",
    "mape": 17.34396231950948,
    "d90_th": 785,
    "d90_em": 881,
    "err_pct": 10.896708286038592,
    "dH_90": -0.07825223337361463,
    "ratio90": 0.3925
  },
  {
    "gamma": 1.0454762537473639,
    "log_A": -2.4338207488763257,
    "R2": 0.997461,
    "theta": 500000,
    "corpus": "mongo",
    "model": "meta-llama/Meta-Llama-3-8B",
    "attn_empirical": {
      "10": 0.0059040391059695845,
      "20": 0.0036818826782594742,
      "30": 0.0025728944933184213,
      "50": 0.0014714971260077114,
      "100": 0.0006953646930115712,
      "200": 0.00037121372909395075,
      "500": 0.00011184491504738818,
      "1000": 6.471107897798195e-05,
      "2000": 3.34952792859945e-05
    },
    "distances_fit": [
      30,
      50,
      100,
      200,
      500,
      1000,
      2000
    ],
    "stem": "meta-llama--Meta-Llama-3-8B_mongo",
    "mape": 5.5592357782201764,
    "d90_th": 718,
    "d90_em": 1037,
    "err_pct": 30.76181292189007,
    "dH_90": -0.0018095444887590362,
    "ratio90": 0.359
  },
  {
    "gamma": 1.060750419523944,
    "log_A": -2.143867119472637,
    "R2": 0.99869,
    "theta": 10000,
    "corpus": "mongo",
    "model": "mistralai/Mistral-7B-v0.1",
    "attn_empirical": {
      "10": 0.006733735190031843,
      "20": 0.004026857788234742,
      "30": 0.0030662473954726015,
      "50": 0.0019213843246042315,
      "100": 0.0009038505519533323,
      "200": 0.00039353982273799675,
      "500": 0.0001694695357905908,
      "1000": 8.260491097138987e-05,
      "2000": 3.443458402115438e-05
    },
    "distances_fit": [
      30,
      50,
      100,
      200,
      500,
      1000,
      2000
    ],
    "stem": "mistralai--Mistral-7B-v0.1_mongo",
    "mape": 5.214559494114403,
    "d90_th": 674,
    "d90_em": 947,
    "err_pct": 28.827877507919748,
    "dH_90": -0.08571614105160623,
    "ratio90": 0.337
  },
  {
    "gamma": 1.1347958464287666,
    "log_A": -0.9640958037685541,
    "R2": 0.976472,
    "theta": 10000,
    "corpus": "random",
    "model": "google/gemma-2-9b-it",
    "attn_empirical": {
      "10": 0.01131202671935575,
      "20": 0.007342287618666887,
      "30": 0.005608218560616176,
      "50": 0.004600468463678327,
      "100": 0.0025041243884091576,
      "200": 0.0011723479800275526,
      "500": 0.00037003465085743104,
      "1000": 0.0001858278461845152,
      "2000": 4.510536613654242e-05
    },
    "distances_fit": [
      30,
      50,
      100,
      200,
      500,
      1000,
      2000
    ],
    "stem": "google--gemma-2-9b-it_random",
    "mape": 23.66528743924567,
    "d90_th": 469,
    "d90_em": 798,
    "err_pct": 41.228070175438596,
    "dH_90": -0.09964902083307159,
    "ratio90": 0.2345
  },
  {
    "gamma": 1.2350013988825523,
    "log_A": -0.8481173688844952,
    "R2": 0.973742,
    "theta": 10000,
    "corpus": "mongo",
    "model": "EleutherAI/pythia-31m",
    "attn_empirical": {
      "10": 0.015075270254164932,
      "20": 0.007979718983923603,
      "30": 0.004943025353131816,
      "50": 0.003909375242526746,
      "100": 0.001335802334417369,
      "200": 0.0006274800656319712,
      "500": 0.0003229697247034993,
      "1000": 0.00010229767169578944,
      "2000": 2.207666448479115e-05
    },
    "distances_fit": [
      30,
      50,
      100,
      200,
      500,
      1000,
      2000
    ],
    "stem": "EleutherAI--pythia-31m_mongo",
    "mape": 24.468814041997238,
    "d90_th": 246,
    "d90_em": 572,
    "err_pct": 56.993006993006986,
    "dH_90": -0.11502902082432076,
    "ratio90": 0.123
  }
]